aboutsummaryrefslogtreecommitdiffstats
path: root/vendor/cloud.google.com/go/bigquery
diff options
context:
space:
mode:
authorDmitry Vyukov <dvyukov@google.com>2017-06-13 20:23:21 +0200
committerDmitry Vyukov <dvyukov@google.com>2017-06-13 20:23:21 +0200
commitb41e96b421acda1761e9ba2ee17da16efad436cd (patch)
treeb6200bf1c3665161b6d76453bb56f7bd5ce32f42 /vendor/cloud.google.com/go/bigquery
parentf58147fb5ea43427d38c2a4437f35647138fbf9d (diff)
vendor: switch from dep to godep
dep tool vendored too much code (100MB) including tests and unused packages. godep vendored significantly less (12MB) without tests and unused packages. The main advantage is that pre-Go1.9 toolchain does not run tests of all vendor packages now.
Diffstat (limited to 'vendor/cloud.google.com/go/bigquery')
-rw-r--r--vendor/cloud.google.com/go/bigquery/bigquery.go85
-rw-r--r--vendor/cloud.google.com/go/bigquery/copy.go74
-rw-r--r--vendor/cloud.google.com/go/bigquery/copy_test.go136
-rw-r--r--vendor/cloud.google.com/go/bigquery/create_table_test.go103
-rw-r--r--vendor/cloud.google.com/go/bigquery/dataset.go188
-rw-r--r--vendor/cloud.google.com/go/bigquery/dataset_test.go156
-rw-r--r--vendor/cloud.google.com/go/bigquery/doc.go295
-rw-r--r--vendor/cloud.google.com/go/bigquery/error.go82
-rw-r--r--vendor/cloud.google.com/go/bigquery/error_test.go109
-rw-r--r--vendor/cloud.google.com/go/bigquery/examples_test.go652
-rw-r--r--vendor/cloud.google.com/go/bigquery/extract.go76
-rw-r--r--vendor/cloud.google.com/go/bigquery/extract_test.go102
-rw-r--r--vendor/cloud.google.com/go/bigquery/file.go172
-rw-r--r--vendor/cloud.google.com/go/bigquery/file_test.go90
-rw-r--r--vendor/cloud.google.com/go/bigquery/gcs.go68
-rw-r--r--vendor/cloud.google.com/go/bigquery/integration_test.go1038
-rw-r--r--vendor/cloud.google.com/go/bigquery/iterator.go154
-rw-r--r--vendor/cloud.google.com/go/bigquery/iterator_test.go363
-rw-r--r--vendor/cloud.google.com/go/bigquery/job.go326
-rw-r--r--vendor/cloud.google.com/go/bigquery/load.go86
-rw-r--r--vendor/cloud.google.com/go/bigquery/load_test.go229
-rw-r--r--vendor/cloud.google.com/go/bigquery/params.go265
-rw-r--r--vendor/cloud.google.com/go/bigquery/params_test.go262
-rw-r--r--vendor/cloud.google.com/go/bigquery/query.go206
-rw-r--r--vendor/cloud.google.com/go/bigquery/query_test.go305
-rw-r--r--vendor/cloud.google.com/go/bigquery/read_test.go260
-rw-r--r--vendor/cloud.google.com/go/bigquery/schema.go315
-rw-r--r--vendor/cloud.google.com/go/bigquery/schema_test.go796
-rw-r--r--vendor/cloud.google.com/go/bigquery/service.go731
-rw-r--r--vendor/cloud.google.com/go/bigquery/service_test.go83
-rw-r--r--vendor/cloud.google.com/go/bigquery/table.go253
-rw-r--r--vendor/cloud.google.com/go/bigquery/uploader.go177
-rw-r--r--vendor/cloud.google.com/go/bigquery/uploader_test.go279
-rw-r--r--vendor/cloud.google.com/go/bigquery/utils_test.go47
-rw-r--r--vendor/cloud.google.com/go/bigquery/value.go657
-rw-r--r--vendor/cloud.google.com/go/bigquery/value_test.go885
36 files changed, 0 insertions, 10105 deletions
diff --git a/vendor/cloud.google.com/go/bigquery/bigquery.go b/vendor/cloud.google.com/go/bigquery/bigquery.go
deleted file mode 100644
index 6eb3fc7a4..000000000
--- a/vendor/cloud.google.com/go/bigquery/bigquery.go
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-// TODO(mcgreevy): support dry-run mode when creating jobs.
-
-import (
- "fmt"
-
- "google.golang.org/api/option"
- "google.golang.org/api/transport"
-
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-const prodAddr = "https://www.googleapis.com/bigquery/v2/"
-
-// ExternalData is a table which is stored outside of BigQuery. It is implemented by GCSReference.
-type ExternalData interface {
- externalDataConfig() bq.ExternalDataConfiguration
-}
-
-const Scope = "https://www.googleapis.com/auth/bigquery"
-const userAgent = "gcloud-golang-bigquery/20160429"
-
-// Client may be used to perform BigQuery operations.
-type Client struct {
- service service
- projectID string
-}
-
-// NewClient constructs a new Client which can perform BigQuery operations.
-// Operations performed via the client are billed to the specified GCP project.
-func NewClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*Client, error) {
- o := []option.ClientOption{
- option.WithEndpoint(prodAddr),
- option.WithScopes(Scope),
- option.WithUserAgent(userAgent),
- }
- o = append(o, opts...)
- httpClient, endpoint, err := transport.NewHTTPClient(ctx, o...)
- if err != nil {
- return nil, fmt.Errorf("dialing: %v", err)
- }
-
- s, err := newBigqueryService(httpClient, endpoint)
- if err != nil {
- return nil, fmt.Errorf("constructing bigquery client: %v", err)
- }
-
- c := &Client{
- service: s,
- projectID: projectID,
- }
- return c, nil
-}
-
-// Close closes any resources held by the client.
-// Close should be called when the client is no longer needed.
-// It need not be called at program exit.
-func (c *Client) Close() error {
- return nil
-}
-
-func (c *Client) insertJob(ctx context.Context, conf *insertJobConf) (*Job, error) {
- job, err := c.service.insertJob(ctx, c.projectID, conf)
- if err != nil {
- return nil, err
- }
- job.c = c
- return job, nil
-}
diff --git a/vendor/cloud.google.com/go/bigquery/copy.go b/vendor/cloud.google.com/go/bigquery/copy.go
deleted file mode 100644
index 726999f69..000000000
--- a/vendor/cloud.google.com/go/bigquery/copy.go
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// CopyConfig holds the configuration for a copy job.
-type CopyConfig struct {
- // JobID is the ID to use for the copy job. If unset, a job ID will be automatically created.
- JobID string
-
- // Srcs are the tables from which data will be copied.
- Srcs []*Table
-
- // Dst is the table into which the data will be copied.
- Dst *Table
-
- // CreateDisposition specifies the circumstances under which the destination table will be created.
- // The default is CreateIfNeeded.
- CreateDisposition TableCreateDisposition
-
- // WriteDisposition specifies how existing data in the destination table is treated.
- // The default is WriteAppend.
- WriteDisposition TableWriteDisposition
-}
-
-// A Copier copies data into a BigQuery table from one or more BigQuery tables.
-type Copier struct {
- CopyConfig
- c *Client
-}
-
-// CopierFrom returns a Copier which can be used to copy data into a
-// BigQuery table from one or more BigQuery tables.
-// The returned Copier may optionally be further configured before its Run method is called.
-func (t *Table) CopierFrom(srcs ...*Table) *Copier {
- return &Copier{
- c: t.c,
- CopyConfig: CopyConfig{
- Srcs: srcs,
- Dst: t,
- },
- }
-}
-
-// Run initiates a copy job.
-func (c *Copier) Run(ctx context.Context) (*Job, error) {
- conf := &bq.JobConfigurationTableCopy{
- CreateDisposition: string(c.CreateDisposition),
- WriteDisposition: string(c.WriteDisposition),
- DestinationTable: c.Dst.tableRefProto(),
- }
- for _, t := range c.Srcs {
- conf.SourceTables = append(conf.SourceTables, t.tableRefProto())
- }
- job := &bq.Job{Configuration: &bq.JobConfiguration{Copy: conf}}
- setJobRef(job, c.JobID, c.c.projectID)
- return c.c.insertJob(ctx, &insertJobConf{job: job})
-}
diff --git a/vendor/cloud.google.com/go/bigquery/copy_test.go b/vendor/cloud.google.com/go/bigquery/copy_test.go
deleted file mode 100644
index f55ce28aa..000000000
--- a/vendor/cloud.google.com/go/bigquery/copy_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
-
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func defaultCopyJob() *bq.Job {
- return &bq.Job{
- Configuration: &bq.JobConfiguration{
- Copy: &bq.JobConfigurationTableCopy{
- DestinationTable: &bq.TableReference{
- ProjectId: "d-project-id",
- DatasetId: "d-dataset-id",
- TableId: "d-table-id",
- },
- SourceTables: []*bq.TableReference{
- {
- ProjectId: "s-project-id",
- DatasetId: "s-dataset-id",
- TableId: "s-table-id",
- },
- },
- },
- },
- }
-}
-
-func TestCopy(t *testing.T) {
- testCases := []struct {
- dst *Table
- srcs []*Table
- config CopyConfig
- want *bq.Job
- }{
- {
- dst: &Table{
- ProjectID: "d-project-id",
- DatasetID: "d-dataset-id",
- TableID: "d-table-id",
- },
- srcs: []*Table{
- {
- ProjectID: "s-project-id",
- DatasetID: "s-dataset-id",
- TableID: "s-table-id",
- },
- },
- want: defaultCopyJob(),
- },
- {
- dst: &Table{
- ProjectID: "d-project-id",
- DatasetID: "d-dataset-id",
- TableID: "d-table-id",
- },
- srcs: []*Table{
- {
- ProjectID: "s-project-id",
- DatasetID: "s-dataset-id",
- TableID: "s-table-id",
- },
- },
- config: CopyConfig{
- CreateDisposition: CreateNever,
- WriteDisposition: WriteTruncate,
- },
- want: func() *bq.Job {
- j := defaultCopyJob()
- j.Configuration.Copy.CreateDisposition = "CREATE_NEVER"
- j.Configuration.Copy.WriteDisposition = "WRITE_TRUNCATE"
- return j
- }(),
- },
- {
- dst: &Table{
- ProjectID: "d-project-id",
- DatasetID: "d-dataset-id",
- TableID: "d-table-id",
- },
- srcs: []*Table{
- {
- ProjectID: "s-project-id",
- DatasetID: "s-dataset-id",
- TableID: "s-table-id",
- },
- },
- config: CopyConfig{JobID: "job-id"},
- want: func() *bq.Job {
- j := defaultCopyJob()
- j.JobReference = &bq.JobReference{
- JobId: "job-id",
- ProjectId: "client-project-id",
- }
- return j
- }(),
- },
- }
-
- for _, tc := range testCases {
- s := &testService{}
- c := &Client{
- service: s,
- projectID: "client-project-id",
- }
- tc.dst.c = c
- copier := tc.dst.CopierFrom(tc.srcs...)
- tc.config.Srcs = tc.srcs
- tc.config.Dst = tc.dst
- copier.CopyConfig = tc.config
- if _, err := copier.Run(context.Background()); err != nil {
- t.Errorf("err calling Run: %v", err)
- continue
- }
- if !reflect.DeepEqual(s.Job, tc.want) {
- t.Errorf("copying: got:\n%v\nwant:\n%v", s.Job, tc.want)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/create_table_test.go b/vendor/cloud.google.com/go/bigquery/create_table_test.go
deleted file mode 100644
index 2a7d9d10f..000000000
--- a/vendor/cloud.google.com/go/bigquery/create_table_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
- "time"
-
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-type createTableRecorder struct {
- conf *createTableConf
- service
-}
-
-func (rec *createTableRecorder) createTable(ctx context.Context, conf *createTableConf) error {
- rec.conf = conf
- return nil
-}
-
-func TestCreateTableOptions(t *testing.T) {
- s := &createTableRecorder{}
- c := &Client{
- projectID: "p",
- service: s,
- }
- ds := c.Dataset("d")
- table := ds.Table("t")
- exp := time.Now()
- q := "query"
- if err := table.Create(context.Background(), TableExpiration(exp), ViewQuery(q), UseStandardSQL()); err != nil {
- t.Fatalf("err calling Table.Create: %v", err)
- }
- want := createTableConf{
- projectID: "p",
- datasetID: "d",
- tableID: "t",
- expiration: exp,
- viewQuery: q,
- useStandardSQL: true,
- }
- if !reflect.DeepEqual(*s.conf, want) {
- t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
- }
-
- sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
- if err := table.Create(context.Background(), TableExpiration(exp), sc); err != nil {
- t.Fatalf("err calling Table.Create: %v", err)
- }
- want = createTableConf{
- projectID: "p",
- datasetID: "d",
- tableID: "t",
- expiration: exp,
- // No need for an elaborate schema, that is tested in schema_test.go.
- schema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
- },
- },
- }
- if !reflect.DeepEqual(*s.conf, want) {
- t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
- }
-
- partitionCases := []struct {
- timePartitioning TimePartitioning
- expectedExpiration time.Duration
- }{
- {TimePartitioning{}, time.Duration(0)},
- {TimePartitioning{time.Second}, time.Second},
- }
-
- for _, c := range partitionCases {
- if err := table.Create(context.Background(), c.timePartitioning); err != nil {
- t.Fatalf("err calling Table.Create: %v", err)
- }
- want = createTableConf{
- projectID: "p",
- datasetID: "d",
- tableID: "t",
- timePartitioning: &TimePartitioning{c.expectedExpiration},
- }
- if !reflect.DeepEqual(*s.conf, want) {
- t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/dataset.go b/vendor/cloud.google.com/go/bigquery/dataset.go
deleted file mode 100644
index 80eacf659..000000000
--- a/vendor/cloud.google.com/go/bigquery/dataset.go
+++ /dev/null
@@ -1,188 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "time"
-
- "golang.org/x/net/context"
- "google.golang.org/api/iterator"
-)
-
-// Dataset is a reference to a BigQuery dataset.
-type Dataset struct {
- ProjectID string
- DatasetID string
- c *Client
-}
-
-type DatasetMetadata struct {
- CreationTime time.Time
- LastModifiedTime time.Time // When the dataset or any of its tables were modified.
- DefaultTableExpiration time.Duration
- Description string // The user-friendly description of this table.
- Name string // The user-friendly name for this table.
- ID string
- Location string // The geo location of the dataset.
- Labels map[string]string // User-provided labels.
- // TODO(jba): access rules
-}
-
-// Dataset creates a handle to a BigQuery dataset in the client's project.
-func (c *Client) Dataset(id string) *Dataset {
- return c.DatasetInProject(c.projectID, id)
-}
-
-// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
-func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
- return &Dataset{
- ProjectID: projectID,
- DatasetID: datasetID,
- c: c,
- }
-}
-
-// Create creates a dataset in the BigQuery service. An error will be returned
-// if the dataset already exists.
-func (d *Dataset) Create(ctx context.Context) error {
- return d.c.service.insertDataset(ctx, d.DatasetID, d.ProjectID)
-}
-
-// Delete deletes the dataset.
-func (d *Dataset) Delete(ctx context.Context) error {
- return d.c.service.deleteDataset(ctx, d.DatasetID, d.ProjectID)
-}
-
-// Metadata fetches the metadata for the dataset.
-func (d *Dataset) Metadata(ctx context.Context) (*DatasetMetadata, error) {
- return d.c.service.getDatasetMetadata(ctx, d.ProjectID, d.DatasetID)
-}
-
-// Table creates a handle to a BigQuery table in the dataset.
-// To determine if a table exists, call Table.Metadata.
-// If the table does not already exist, use Table.Create to create it.
-func (d *Dataset) Table(tableID string) *Table {
- return &Table{ProjectID: d.ProjectID, DatasetID: d.DatasetID, TableID: tableID, c: d.c}
-}
-
-// Tables returns an iterator over the tables in the Dataset.
-func (d *Dataset) Tables(ctx context.Context) *TableIterator {
- it := &TableIterator{
- ctx: ctx,
- dataset: d,
- }
- it.pageInfo, it.nextFunc = iterator.NewPageInfo(
- it.fetch,
- func() int { return len(it.tables) },
- func() interface{} { b := it.tables; it.tables = nil; return b })
- return it
-}
-
-// A TableIterator is an iterator over Tables.
-type TableIterator struct {
- ctx context.Context
- dataset *Dataset
- tables []*Table
- pageInfo *iterator.PageInfo
- nextFunc func() error
-}
-
-// Next returns the next result. Its second return value is Done if there are
-// no more results. Once Next returns Done, all subsequent calls will return
-// Done.
-func (it *TableIterator) Next() (*Table, error) {
- if err := it.nextFunc(); err != nil {
- return nil, err
- }
- t := it.tables[0]
- it.tables = it.tables[1:]
- return t, nil
-}
-
-// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
-func (it *TableIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
-
-func (it *TableIterator) fetch(pageSize int, pageToken string) (string, error) {
- tables, tok, err := it.dataset.c.service.listTables(it.ctx, it.dataset.ProjectID, it.dataset.DatasetID, pageSize, pageToken)
- if err != nil {
- return "", err
- }
- for _, t := range tables {
- t.c = it.dataset.c
- it.tables = append(it.tables, t)
- }
- return tok, nil
-}
-
-// Datasets returns an iterator over the datasets in the Client's project.
-func (c *Client) Datasets(ctx context.Context) *DatasetIterator {
- return c.DatasetsInProject(ctx, c.projectID)
-}
-
-// DatasetsInProject returns an iterator over the datasets in the provided project.
-func (c *Client) DatasetsInProject(ctx context.Context, projectID string) *DatasetIterator {
- it := &DatasetIterator{
- ctx: ctx,
- c: c,
- projectID: projectID,
- }
- it.pageInfo, it.nextFunc = iterator.NewPageInfo(
- it.fetch,
- func() int { return len(it.items) },
- func() interface{} { b := it.items; it.items = nil; return b })
- return it
-}
-
-// DatasetIterator iterates over the datasets in a project.
-type DatasetIterator struct {
- // ListHidden causes hidden datasets to be listed when set to true.
- ListHidden bool
-
- // Filter restricts the datasets returned by label. The filter syntax is described in
- // https://cloud.google.com/bigquery/docs/labeling-datasets#filtering_datasets_using_labels
- Filter string
-
- ctx context.Context
- projectID string
- c *Client
- pageInfo *iterator.PageInfo
- nextFunc func() error
- items []*Dataset
-}
-
-// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
-func (it *DatasetIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
-
-func (it *DatasetIterator) Next() (*Dataset, error) {
- if err := it.nextFunc(); err != nil {
- return nil, err
- }
- item := it.items[0]
- it.items = it.items[1:]
- return item, nil
-}
-
-func (it *DatasetIterator) fetch(pageSize int, pageToken string) (string, error) {
- datasets, nextPageToken, err := it.c.service.listDatasets(it.ctx, it.projectID,
- pageSize, pageToken, it.ListHidden, it.Filter)
- if err != nil {
- return "", err
- }
- for _, d := range datasets {
- d.c = it.c
- it.items = append(it.items, d)
- }
- return nextPageToken, nil
-}
diff --git a/vendor/cloud.google.com/go/bigquery/dataset_test.go b/vendor/cloud.google.com/go/bigquery/dataset_test.go
deleted file mode 100644
index 01b8ebdb2..000000000
--- a/vendor/cloud.google.com/go/bigquery/dataset_test.go
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "strconv"
- "testing"
-
- "golang.org/x/net/context"
- itest "google.golang.org/api/iterator/testing"
-)
-
-// readServiceStub services read requests by returning data from an in-memory list of values.
-type listTablesServiceStub struct {
- expectedProject, expectedDataset string
- tables []*Table
- service
-}
-
-func (s *listTablesServiceStub) listTables(ctx context.Context, projectID, datasetID string, pageSize int, pageToken string) ([]*Table, string, error) {
- if projectID != s.expectedProject {
- return nil, "", errors.New("wrong project id")
- }
- if datasetID != s.expectedDataset {
- return nil, "", errors.New("wrong dataset id")
- }
- const maxPageSize = 2
- if pageSize <= 0 || pageSize > maxPageSize {
- pageSize = maxPageSize
- }
- start := 0
- if pageToken != "" {
- var err error
- start, err = strconv.Atoi(pageToken)
- if err != nil {
- return nil, "", err
- }
- }
- end := start + pageSize
- if end > len(s.tables) {
- end = len(s.tables)
- }
- nextPageToken := ""
- if end < len(s.tables) {
- nextPageToken = strconv.Itoa(end)
- }
- return s.tables[start:end], nextPageToken, nil
-}
-
-func TestTables(t *testing.T) {
- t1 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t1"}
- t2 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t2"}
- t3 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t3"}
- allTables := []*Table{t1, t2, t3}
- c := &Client{
- service: &listTablesServiceStub{
- expectedProject: "x",
- expectedDataset: "y",
- tables: allTables,
- },
- projectID: "x",
- }
- msg, ok := itest.TestIterator(allTables,
- func() interface{} { return c.Dataset("y").Tables(context.Background()) },
- func(it interface{}) (interface{}, error) { return it.(*TableIterator).Next() })
- if !ok {
- t.Error(msg)
- }
-}
-
-type listDatasetsFake struct {
- service
-
- projectID string
- datasets []*Dataset
- hidden map[*Dataset]bool
-}
-
-func (df *listDatasetsFake) listDatasets(_ context.Context, projectID string, pageSize int, pageToken string, listHidden bool, filter string) ([]*Dataset, string, error) {
- const maxPageSize = 2
- if pageSize <= 0 || pageSize > maxPageSize {
- pageSize = maxPageSize
- }
- if filter != "" {
- return nil, "", errors.New("filter not supported")
- }
- if projectID != df.projectID {
- return nil, "", errors.New("bad project ID")
- }
- start := 0
- if pageToken != "" {
- var err error
- start, err = strconv.Atoi(pageToken)
- if err != nil {
- return nil, "", err
- }
- }
- var (
- i int
- result []*Dataset
- nextPageToken string
- )
- for i = start; len(result) < pageSize && i < len(df.datasets); i++ {
- if df.hidden[df.datasets[i]] && !listHidden {
- continue
- }
- result = append(result, df.datasets[i])
- }
- if i < len(df.datasets) {
- nextPageToken = strconv.Itoa(i)
- }
- return result, nextPageToken, nil
-}
-
-func TestDatasets(t *testing.T) {
- service := &listDatasetsFake{projectID: "p"}
- client := &Client{service: service}
- datasets := []*Dataset{
- {"p", "a", client},
- {"p", "b", client},
- {"p", "hidden", client},
- {"p", "c", client},
- }
- service.datasets = datasets
- service.hidden = map[*Dataset]bool{datasets[2]: true}
- c := &Client{
- projectID: "p",
- service: service,
- }
- msg, ok := itest.TestIterator(datasets,
- func() interface{} { it := c.Datasets(context.Background()); it.ListHidden = true; return it },
- func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
- if !ok {
- t.Fatalf("ListHidden=true: %s", msg)
- }
-
- msg, ok = itest.TestIterator([]*Dataset{datasets[0], datasets[1], datasets[3]},
- func() interface{} { it := c.Datasets(context.Background()); it.ListHidden = false; return it },
- func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
- if !ok {
- t.Fatalf("ListHidden=false: %s", msg)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/doc.go b/vendor/cloud.google.com/go/bigquery/doc.go
deleted file mode 100644
index f644792bc..000000000
--- a/vendor/cloud.google.com/go/bigquery/doc.go
+++ /dev/null
@@ -1,295 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/*
-Package bigquery provides a client for the BigQuery service.
-
-Note: This package is in beta. Some backwards-incompatible changes may occur.
-
-The following assumes a basic familiarity with BigQuery concepts.
-See https://cloud.google.com/bigquery/docs.
-
-
-Creating a Client
-
-To start working with this package, create a client:
-
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, projectID)
- if err != nil {
- // TODO: Handle error.
- }
-
-Querying
-
-To query existing tables, create a Query and call its Read method:
-
- q := client.Query(`
- SELECT year, SUM(number) as num
- FROM [bigquery-public-data:usa_names.usa_1910_2013]
- WHERE name = "William"
- GROUP BY year
- ORDER BY year
- `)
- it, err := q.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
-
-Then iterate through the resulting rows. You can store a row using
-anything that implements the ValueLoader interface, or with a slice or map of bigquery.Value.
-A slice is simplest:
-
- for {
- var values []bigquery.Value
- err := it.Next(&values)
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(values)
- }
-
-You can also use a struct whose exported fields match the query:
-
- type Count struct {
- Year int
- Num int
- }
- for {
- var c Count
- err := it.Next(&c)
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(c)
- }
-
-You can also start the query running and get the results later.
-Create the query as above, but call Run instead of Read. This returns a Job,
-which represents an asychronous operation.
-
- job, err := q.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
-
-Get the job's ID, a printable string. You can save this string to retrieve
-the results at a later time, even in another process.
-
- jobID := job.ID()
- fmt.Printf("The job ID is %s\n", jobID)
-
-To retrieve the job's results from the ID, first look up the Job:
-
- job, err = client.JobFromID(ctx, jobID)
- if err != nil {
- // TODO: Handle error.
- }
-
-Use the Job.Read method to obtain an iterator, and loop over the rows.
-Query.Read is just a convenience method that combines Query.Run and Job.Read.
-
- it, err = job.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- // Proceed with iteration as above.
-
-Datasets and Tables
-
-You can refer to datasets in the client's project with the Dataset method, and
-in other projects with the DatasetInProject method:
-
- myDataset := client.Dataset("my_dataset")
- yourDataset := client.DatasetInProject("your-project-id", "your_dataset")
-
-These methods create references to datasets, not the datasets themselves. You can have
-a dataset reference even if the dataset doesn't exist yet. Use Dataset.Create to
-create a dataset from a reference:
-
- if err := myDataset.Create(ctx); err != nil {
- // TODO: Handle error.
- }
-
-You can refer to tables with Dataset.Table. Like bigquery.Dataset, bigquery.Table is a reference
-to an object in BigQuery that may or may not exist.
-
- table := myDataset.Table("my_table")
-
-You can create, delete and update the metadata of tables with methods on Table.
-Table.Create supports a few options. For instance, you could create a temporary table with:
-
- err = myDataset.Table("temp").Create(ctx, bigquery.TableExpiration(time.Now().Add(1*time.Hour)))
- if err != nil {
- // TODO: Handle error.
- }
-
-We'll see how to create a table with a schema in the next section.
-
-Schemas
-
-There are two ways to construct schemas with this package.
-You can build a schema by hand, like so:
-
- schema1 := bigquery.Schema{
- &bigquery.FieldSchema{Name: "Name", Required: true, Type: bigquery.StringFieldType},
- &bigquery.FieldSchema{Name: "Grades", Repeated: true, Type: bigquery.IntegerFieldType},
- }
-
-Or you can infer the schema from a struct:
-
- type student struct {
- Name string
- Grades []int
- }
- schema2, err := bigquery.InferSchema(student{})
- if err != nil {
- // TODO: Handle error.
- }
- // schema1 and schema2 are identical.
-
-Struct inference supports tags like those of the encoding/json package,
-so you can change names or ignore fields:
-
- type student2 struct {
- Name string `bigquery:"full_name"`
- Grades []int
- Secret string `bigquery:"-"`
- }
- schema3, err := bigquery.InferSchema(student2{})
- if err != nil {
- // TODO: Handle error.
- }
- // schema3 has fields "full_name" and "Grade".
-
-Having constructed a schema, you can pass it to Table.Create as an option:
-
- if err := table.Create(ctx, schema1); err != nil {
- // TODO: Handle error.
- }
-
-Copying
-
-You can copy one or more tables to another table. Begin by constructing a Copier
-describing the copy. Then set any desired copy options, and finally call Run to get a Job:
-
- copier := myDataset.Table("dest").CopierFrom(myDataset.Table("src"))
- copier.WriteDisposition = bigquery.WriteTruncate
- job, err = copier.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
-
-You can chain the call to Run if you don't want to set options:
-
- job, err = myDataset.Table("dest").CopierFrom(myDataset.Table("src")).Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
-
-You can wait for your job to complete:
-
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
-
-Job.Wait polls with exponential backoff. You can also poll yourself, if you
-wish:
-
- for {
- status, err := job.Status(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Done() {
- if status.Err() != nil {
- log.Fatalf("Job failed with error %v", status.Err())
- }
- break
- }
- time.Sleep(pollInterval)
- }
-
-Loading and Uploading
-
-There are two ways to populate a table with this package: load the data from a Google Cloud Storage
-object, or upload rows directly from your program.
-
-For loading, first create a GCSReference, configuring it if desired. Then make a Loader, optionally configure
-it as well, and call its Run method.
-
- gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
- gcsRef.AllowJaggedRows = true
- loader := myDataset.Table("dest").LoaderFrom(gcsRef)
- loader.CreateDisposition = bigquery.CreateNever
- job, err = loader.Run(ctx)
- // Poll the job for completion if desired, as above.
-
-To upload, first define a type that implements the ValueSaver interface, which has a single method named Save.
-Then create an Uploader, and call its Put method with a slice of values.
-
- u := table.Uploader()
- // Item implements the ValueSaver interface.
- items := []*Item{
- {Name: "n1", Size: 32.6, Count: 7},
- {Name: "n2", Size: 4, Count: 2},
- {Name: "n3", Size: 101.5, Count: 1},
- }
- if err := u.Put(ctx, items); err != nil {
- // TODO: Handle error.
- }
-
-You can also upload a struct that doesn't implement ValueSaver. Use the StructSaver type
-to specify the schema and insert ID by hand, or just supply the struct or struct pointer
-directly and the schema will be inferred:
-
- type Item2 struct {
- Name string
- Size float64
- Count int
- }
- // Item implements the ValueSaver interface.
- items2 := []*Item2{
- {Name: "n1", Size: 32.6, Count: 7},
- {Name: "n2", Size: 4, Count: 2},
- {Name: "n3", Size: 101.5, Count: 1},
- }
- if err := u.Put(ctx, items2); err != nil {
- // TODO: Handle error.
- }
-
-Extracting
-
-If you've been following so far, extracting data from a BigQuery table
-into a Google Cloud Storage object will feel familiar. First create an
-Extractor, then optionally configure it, and lastly call its Run method.
-
- extractor := table.ExtractorTo(gcsRef)
- extractor.DisableHeader = true
- job, err = extractor.Run(ctx)
- // Poll the job for completion if desired, as above.
-
-Authentication
-
-See examples of authorization and authentication at
-https://godoc.org/cloud.google.com/go#pkg-examples.
-*/
-package bigquery // import "cloud.google.com/go/bigquery"
diff --git a/vendor/cloud.google.com/go/bigquery/error.go b/vendor/cloud.google.com/go/bigquery/error.go
deleted file mode 100644
index b59ac6e6e..000000000
--- a/vendor/cloud.google.com/go/bigquery/error.go
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "fmt"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// An Error contains detailed information about a failed bigquery operation.
-type Error struct {
- // Mirrors bq.ErrorProto, but drops DebugInfo
- Location, Message, Reason string
-}
-
-func (e Error) Error() string {
- return fmt.Sprintf("{Location: %q; Message: %q; Reason: %q}", e.Location, e.Message, e.Reason)
-}
-
-func errorFromErrorProto(ep *bq.ErrorProto) *Error {
- if ep == nil {
- return nil
- }
- return &Error{
- Location: ep.Location,
- Message: ep.Message,
- Reason: ep.Reason,
- }
-}
-
-// A MultiError contains multiple related errors.
-type MultiError []error
-
-func (m MultiError) Error() string {
- switch len(m) {
- case 0:
- return "(0 errors)"
- case 1:
- return m[0].Error()
- case 2:
- return m[0].Error() + " (and 1 other error)"
- }
- return fmt.Sprintf("%s (and %d other errors)", m[0].Error(), len(m)-1)
-}
-
-// RowInsertionError contains all errors that occurred when attempting to insert a row.
-type RowInsertionError struct {
- InsertID string // The InsertID associated with the affected row.
- RowIndex int // The 0-based index of the affected row in the batch of rows being inserted.
- Errors MultiError
-}
-
-func (e *RowInsertionError) Error() string {
- errFmt := "insertion of row [insertID: %q; insertIndex: %v] failed with error: %s"
- return fmt.Sprintf(errFmt, e.InsertID, e.RowIndex, e.Errors.Error())
-}
-
-// PutMultiError contains an error for each row which was not successfully inserted
-// into a BigQuery table.
-type PutMultiError []RowInsertionError
-
-func (pme PutMultiError) Error() string {
- plural := "s"
- if len(pme) == 1 {
- plural = ""
- }
-
- return fmt.Sprintf("%v row insertion%s failed", len(pme), plural)
-}
diff --git a/vendor/cloud.google.com/go/bigquery/error_test.go b/vendor/cloud.google.com/go/bigquery/error_test.go
deleted file mode 100644
index c0f40bae6..000000000
--- a/vendor/cloud.google.com/go/bigquery/error_test.go
+++ /dev/null
@@ -1,109 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "reflect"
- "strings"
- "testing"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func rowInsertionError(msg string) RowInsertionError {
- return RowInsertionError{Errors: []error{errors.New(msg)}}
-}
-
-func TestPutMultiErrorString(t *testing.T) {
- testCases := []struct {
- errs PutMultiError
- want string
- }{
- {
- errs: PutMultiError{},
- want: "0 row insertions failed",
- },
- {
- errs: PutMultiError{rowInsertionError("a")},
- want: "1 row insertion failed",
- },
- {
- errs: PutMultiError{rowInsertionError("a"), rowInsertionError("b")},
- want: "2 row insertions failed",
- },
- }
-
- for _, tc := range testCases {
- if tc.errs.Error() != tc.want {
- t.Errorf("PutMultiError string: got:\n%v\nwant:\n%v", tc.errs.Error(), tc.want)
- }
- }
-}
-
-func TestMultiErrorString(t *testing.T) {
- testCases := []struct {
- errs MultiError
- want string
- }{
- {
- errs: MultiError{},
- want: "(0 errors)",
- },
- {
- errs: MultiError{errors.New("a")},
- want: "a",
- },
- {
- errs: MultiError{errors.New("a"), errors.New("b")},
- want: "a (and 1 other error)",
- },
- {
- errs: MultiError{errors.New("a"), errors.New("b"), errors.New("c")},
- want: "a (and 2 other errors)",
- },
- }
-
- for _, tc := range testCases {
- if tc.errs.Error() != tc.want {
- t.Errorf("PutMultiError string: got:\n%v\nwant:\n%v", tc.errs.Error(), tc.want)
- }
- }
-}
-
-func TestErrorFromErrorProto(t *testing.T) {
- for _, test := range []struct {
- in *bq.ErrorProto
- want *Error
- }{
- {nil, nil},
- {
- in: &bq.ErrorProto{Location: "L", Message: "M", Reason: "R"},
- want: &Error{Location: "L", Message: "M", Reason: "R"},
- },
- } {
- if got := errorFromErrorProto(test.in); !reflect.DeepEqual(got, test.want) {
- t.Errorf("%v: got %v, want %v", test.in, got, test.want)
- }
- }
-}
-
-func TestErrorString(t *testing.T) {
- e := &Error{Location: "<L>", Message: "<M>", Reason: "<R>"}
- got := e.Error()
- if !strings.Contains(got, "<L>") || !strings.Contains(got, "<M>") || !strings.Contains(got, "<R>") {
- t.Errorf(`got %q, expected to see "<L>", "<M>" and "<R>"`, got)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/examples_test.go b/vendor/cloud.google.com/go/bigquery/examples_test.go
deleted file mode 100644
index 5ceafff71..000000000
--- a/vendor/cloud.google.com/go/bigquery/examples_test.go
+++ /dev/null
@@ -1,652 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery_test
-
-import (
- "fmt"
- "os"
-
- "cloud.google.com/go/bigquery"
- "golang.org/x/net/context"
- "google.golang.org/api/iterator"
-)
-
-func ExampleNewClient() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- _ = client // TODO: Use client.
-}
-
-func ExampleClient_Dataset() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- ds := client.Dataset("my_dataset")
- fmt.Println(ds)
-}
-
-func ExampleClient_DatasetInProject() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- ds := client.DatasetInProject("their-project-id", "their-dataset")
- fmt.Println(ds)
-}
-
-func ExampleClient_Datasets() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.Datasets(ctx)
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func ExampleClient_DatasetsInProject() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.DatasetsInProject(ctx, "their-project-id")
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func getJobID() string { return "" }
-
-func ExampleClient_JobFromID() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- jobID := getJobID() // Get a job ID using Job.ID, the console or elsewhere.
- job, err := client.JobFromID(ctx, jobID)
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(job)
-}
-
-func ExampleNewGCSReference() {
- gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
- fmt.Println(gcsRef)
-}
-
-func ExampleClient_Query() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- q := client.Query("select name, num from t1")
- q.DefaultProjectID = "project-id"
- // TODO: set other options on the Query.
- // TODO: Call Query.Run or Query.Read.
-}
-
-func ExampleClient_Query_parameters() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- q := client.Query("select num from t1 where name = @user")
- q.Parameters = []bigquery.QueryParameter{
- {Name: "user", Value: "Elizabeth"},
- }
- // TODO: set other options on the Query.
- // TODO: Call Query.Run or Query.Read.
-}
-
-func ExampleQuery_Read() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- q := client.Query("select name, num from t1")
- it, err := q.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func ExampleRowIterator_Next() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- q := client.Query("select name, num from t1")
- it, err := q.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- for {
- var row []bigquery.Value
- err := it.Next(&row)
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(row)
- }
-}
-
-func ExampleRowIterator_Next_struct() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
-
- type score struct {
- Name string
- Num int
- }
-
- q := client.Query("select name, num from t1")
- it, err := q.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- for {
- var s score
- err := it.Next(&s)
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(s)
- }
-}
-
-func ExampleJob_Read() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- q := client.Query("select name, num from t1")
- // Call Query.Run to get a Job, then call Read on the job.
- // Note: Query.Read is a shorthand for this.
- job, err := q.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- it, err := job.Read(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func ExampleJob_Wait() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- ds := client.Dataset("my_dataset")
- job, err := ds.Table("t1").CopierFrom(ds.Table("t2")).Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Err() != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleDataset_Create() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- if err := client.Dataset("my_dataset").Create(ctx); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleDataset_Delete() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- if err := client.Dataset("my_dataset").Delete(ctx); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleDataset_Metadata() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- md, err := client.Dataset("my_dataset").Metadata(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(md)
-}
-
-func ExampleDataset_Table() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- // Table creates a reference to the table. It does not create the actual
- // table in BigQuery; to do so, use Table.Create.
- t := client.Dataset("my_dataset").Table("my_table")
- fmt.Println(t)
-}
-
-func ExampleDataset_Tables() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.Dataset("my_dataset").Tables(ctx)
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func ExampleDatasetIterator_Next() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.Datasets(ctx)
- for {
- ds, err := it.Next()
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(ds)
- }
-}
-
-func ExampleInferSchema() {
- type Item struct {
- Name string
- Size float64
- Count int
- }
- schema, err := bigquery.InferSchema(Item{})
- if err != nil {
- fmt.Println(err)
- // TODO: Handle error.
- }
- for _, fs := range schema {
- fmt.Println(fs.Name, fs.Type)
- }
- // Output:
- // Name STRING
- // Size FLOAT
- // Count INTEGER
-}
-
-func ExampleInferSchema_tags() {
- type Item struct {
- Name string
- Size float64
- Count int `bigquery:"number"`
- Secret []byte `bigquery:"-"`
- }
- schema, err := bigquery.InferSchema(Item{})
- if err != nil {
- fmt.Println(err)
- // TODO: Handle error.
- }
- for _, fs := range schema {
- fmt.Println(fs.Name, fs.Type)
- }
- // Output:
- // Name STRING
- // Size FLOAT
- // number INTEGER
-}
-
-func ExampleTable_Create() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- t := client.Dataset("my_dataset").Table("new-table")
- if err := t.Create(ctx); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_Create_schema() {
- ctx := context.Background()
- // Infer table schema from a Go type.
- schema, err := bigquery.InferSchema(Item{})
- if err != nil {
- // TODO: Handle error.
- }
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- t := client.Dataset("my_dataset").Table("new-table")
- if err := t.Create(ctx, schema); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_Delete() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- if err := client.Dataset("my_dataset").Table("my_table").Delete(ctx); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_Metadata() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- md, err := client.Dataset("my_dataset").Table("my_table").Metadata(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(md)
-}
-
-func ExampleTable_Uploader() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- u := client.Dataset("my_dataset").Table("my_table").Uploader()
- _ = u // TODO: Use u.
-}
-
-func ExampleTable_Uploader_options() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- u := client.Dataset("my_dataset").Table("my_table").Uploader()
- u.SkipInvalidRows = true
- u.IgnoreUnknownValues = true
- _ = u // TODO: Use u.
-}
-
-func ExampleTable_CopierFrom() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- ds := client.Dataset("my_dataset")
- c := ds.Table("combined").CopierFrom(ds.Table("t1"), ds.Table("t2"))
- c.WriteDisposition = bigquery.WriteTruncate
- // TODO: set other options on the Copier.
- job, err := c.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Err() != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_ExtractorTo() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
- gcsRef.FieldDelimiter = ":"
- // TODO: set other options on the GCSReference.
- ds := client.Dataset("my_dataset")
- extractor := ds.Table("my_table").ExtractorTo(gcsRef)
- extractor.DisableHeader = true
- // TODO: set other options on the Extractor.
- job, err := extractor.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Err() != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_LoaderFrom() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- gcsRef := bigquery.NewGCSReference("gs://my-bucket/my-object")
- gcsRef.AllowJaggedRows = true
- // TODO: set other options on the GCSReference.
- ds := client.Dataset("my_dataset")
- loader := ds.Table("my_table").LoaderFrom(gcsRef)
- loader.CreateDisposition = bigquery.CreateNever
- // TODO: set other options on the Loader.
- job, err := loader.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Err() != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_LoaderFrom_reader() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- f, err := os.Open("data.csv")
- if err != nil {
- // TODO: Handle error.
- }
- rs := bigquery.NewReaderSource(f)
- rs.AllowJaggedRows = true
- // TODO: set other options on the GCSReference.
- ds := client.Dataset("my_dataset")
- loader := ds.Table("my_table").LoaderFrom(rs)
- loader.CreateDisposition = bigquery.CreateNever
- // TODO: set other options on the Loader.
- job, err := loader.Run(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- status, err := job.Wait(ctx)
- if err != nil {
- // TODO: Handle error.
- }
- if status.Err() != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleTable_Read() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.Dataset("my_dataset").Table("my_table").Read(ctx)
- _ = it // TODO: iterate using Next or iterator.Pager.
-}
-
-func ExampleTable_Update() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- t := client.Dataset("my_dataset").Table("my_table")
- tm, err := t.Update(ctx, bigquery.TableMetadataToUpdate{
- Description: "my favorite table",
- })
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(tm)
-}
-
-func ExampleTableIterator_Next() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- it := client.Dataset("my_dataset").Tables(ctx)
- for {
- t, err := it.Next()
- if err == iterator.Done {
- break
- }
- if err != nil {
- // TODO: Handle error.
- }
- fmt.Println(t)
- }
-}
-
-type Item struct {
- Name string
- Size float64
- Count int
-}
-
-// Save implements the ValueSaver interface.
-func (i *Item) Save() (map[string]bigquery.Value, string, error) {
- return map[string]bigquery.Value{
- "Name": i.Name,
- "Size": i.Size,
- "Count": i.Count,
- }, "", nil
-}
-
-func ExampleUploader_Put() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- u := client.Dataset("my_dataset").Table("my_table").Uploader()
- // Item implements the ValueSaver interface.
- items := []*Item{
- {Name: "n1", Size: 32.6, Count: 7},
- {Name: "n2", Size: 4, Count: 2},
- {Name: "n3", Size: 101.5, Count: 1},
- }
- if err := u.Put(ctx, items); err != nil {
- // TODO: Handle error.
- }
-}
-
-var schema bigquery.Schema
-
-func ExampleUploader_Put_structSaver() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- u := client.Dataset("my_dataset").Table("my_table").Uploader()
-
- type score struct {
- Name string
- Num int
- }
-
- // Assume schema holds the table's schema.
- savers := []*bigquery.StructSaver{
- {Struct: score{Name: "n1", Num: 12}, Schema: schema, InsertID: "id1"},
- {Struct: score{Name: "n2", Num: 31}, Schema: schema, InsertID: "id2"},
- {Struct: score{Name: "n3", Num: 7}, Schema: schema, InsertID: "id3"},
- }
- if err := u.Put(ctx, savers); err != nil {
- // TODO: Handle error.
- }
-}
-
-func ExampleUploader_Put_struct() {
- ctx := context.Background()
- client, err := bigquery.NewClient(ctx, "project-id")
- if err != nil {
- // TODO: Handle error.
- }
- u := client.Dataset("my_dataset").Table("my_table").Uploader()
-
- type score struct {
- Name string
- Num int
- }
- scores := []score{
- {Name: "n1", Num: 12},
- {Name: "n2", Num: 31},
- {Name: "n3", Num: 7},
- }
- // Schema is inferred from the score type.
- if err := u.Put(ctx, scores); err != nil {
- // TODO: Handle error.
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/extract.go b/vendor/cloud.google.com/go/bigquery/extract.go
deleted file mode 100644
index d4086396b..000000000
--- a/vendor/cloud.google.com/go/bigquery/extract.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// ExtractConfig holds the configuration for an extract job.
-type ExtractConfig struct {
- // JobID is the ID to use for the extract job. If empty, a job ID will be automatically created.
- JobID string
-
- // Src is the table from which data will be extracted.
- Src *Table
-
- // Dst is the destination into which the data will be extracted.
- Dst *GCSReference
-
- // DisableHeader disables the printing of a header row in exported data.
- DisableHeader bool
-}
-
-// An Extractor extracts data from a BigQuery table into Google Cloud Storage.
-type Extractor struct {
- ExtractConfig
- c *Client
-}
-
-// ExtractorTo returns an Extractor which can be used to extract data from a
-// BigQuery table into Google Cloud Storage.
-// The returned Extractor may optionally be further configured before its Run method is called.
-func (t *Table) ExtractorTo(dst *GCSReference) *Extractor {
- return &Extractor{
- c: t.c,
- ExtractConfig: ExtractConfig{
- Src: t,
- Dst: dst,
- },
- }
-}
-
-// Run initiates an extract job.
-func (e *Extractor) Run(ctx context.Context) (*Job, error) {
- conf := &bq.JobConfigurationExtract{}
- job := &bq.Job{Configuration: &bq.JobConfiguration{Extract: conf}}
-
- setJobRef(job, e.JobID, e.c.projectID)
-
- conf.DestinationUris = append([]string{}, e.Dst.uris...)
- conf.Compression = string(e.Dst.Compression)
- conf.DestinationFormat = string(e.Dst.DestinationFormat)
- conf.FieldDelimiter = e.Dst.FieldDelimiter
-
- conf.SourceTable = e.Src.tableRefProto()
-
- if e.DisableHeader {
- f := false
- conf.PrintHeader = &f
- }
-
- return e.c.insertJob(ctx, &insertJobConf{job: job})
-}
diff --git a/vendor/cloud.google.com/go/bigquery/extract_test.go b/vendor/cloud.google.com/go/bigquery/extract_test.go
deleted file mode 100644
index 8079b1829..000000000
--- a/vendor/cloud.google.com/go/bigquery/extract_test.go
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
-
- "golang.org/x/net/context"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func defaultExtractJob() *bq.Job {
- return &bq.Job{
- Configuration: &bq.JobConfiguration{
- Extract: &bq.JobConfigurationExtract{
- SourceTable: &bq.TableReference{
- ProjectId: "project-id",
- DatasetId: "dataset-id",
- TableId: "table-id",
- },
- DestinationUris: []string{"uri"},
- },
- },
- }
-}
-
-func TestExtract(t *testing.T) {
- s := &testService{}
- c := &Client{
- service: s,
- projectID: "project-id",
- }
-
- testCases := []struct {
- dst *GCSReference
- src *Table
- config ExtractConfig
- want *bq.Job
- }{
- {
- dst: defaultGCS(),
- src: c.Dataset("dataset-id").Table("table-id"),
- want: defaultExtractJob(),
- },
- {
- dst: defaultGCS(),
- src: c.Dataset("dataset-id").Table("table-id"),
- config: ExtractConfig{DisableHeader: true},
- want: func() *bq.Job {
- j := defaultExtractJob()
- f := false
- j.Configuration.Extract.PrintHeader = &f
- return j
- }(),
- },
- {
- dst: func() *GCSReference {
- g := NewGCSReference("uri")
- g.Compression = Gzip
- g.DestinationFormat = JSON
- g.FieldDelimiter = "\t"
- return g
- }(),
- src: c.Dataset("dataset-id").Table("table-id"),
- want: func() *bq.Job {
- j := defaultExtractJob()
- j.Configuration.Extract.Compression = "GZIP"
- j.Configuration.Extract.DestinationFormat = "NEWLINE_DELIMITED_JSON"
- j.Configuration.Extract.FieldDelimiter = "\t"
- return j
- }(),
- },
- }
-
- for _, tc := range testCases {
- ext := tc.src.ExtractorTo(tc.dst)
- tc.config.Src = ext.Src
- tc.config.Dst = ext.Dst
- ext.ExtractConfig = tc.config
- if _, err := ext.Run(context.Background()); err != nil {
- t.Errorf("err calling extract: %v", err)
- continue
- }
- if !reflect.DeepEqual(s.Job, tc.want) {
- t.Errorf("extracting: got:\n%v\nwant:\n%v", s.Job, tc.want)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/file.go b/vendor/cloud.google.com/go/bigquery/file.go
deleted file mode 100644
index 0bb3f79d5..000000000
--- a/vendor/cloud.google.com/go/bigquery/file.go
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "io"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// A ReaderSource is a source for a load operation that gets
-// data from an io.Reader.
-type ReaderSource struct {
- r io.Reader
- FileConfig
-}
-
-// NewReaderSource creates a ReaderSource from an io.Reader. You may
-// optionally configure properties on the ReaderSource that describe the
-// data being read, before passing it to Table.LoaderFrom.
-func NewReaderSource(r io.Reader) *ReaderSource {
- return &ReaderSource{r: r}
-}
-
-func (r *ReaderSource) populateInsertJobConfForLoad(conf *insertJobConf) {
- conf.media = r.r
- r.FileConfig.populateLoadConfig(conf.job.Configuration.Load)
-}
-
-// FileConfig contains configuration options that pertain to files, typically
-// text files that require interpretation to be used as a BigQuery table. A
-// file may live in Google Cloud Storage (see GCSReference), or it may be
-// loaded into a table via the Table.LoaderFromReader.
-type FileConfig struct {
- // SourceFormat is the format of the GCS data to be read.
- // Allowed values are: CSV, Avro, JSON, DatastoreBackup. The default is CSV.
- SourceFormat DataFormat
-
- // FieldDelimiter is the separator for fields in a CSV file, used when
- // reading or exporting data. The default is ",".
- FieldDelimiter string
-
- // The number of rows at the top of a CSV file that BigQuery will skip when
- // reading data.
- SkipLeadingRows int64
-
- // AllowJaggedRows causes missing trailing optional columns to be tolerated
- // when reading CSV data. Missing values are treated as nulls.
- AllowJaggedRows bool
-
- // AllowQuotedNewlines sets whether quoted data sections containing
- // newlines are allowed when reading CSV data.
- AllowQuotedNewlines bool
-
- // Indicates if we should automatically infer the options and
- // schema for CSV and JSON sources.
- AutoDetect bool
-
- // Encoding is the character encoding of data to be read.
- Encoding Encoding
-
- // MaxBadRecords is the maximum number of bad records that will be ignored
- // when reading data.
- MaxBadRecords int64
-
- // IgnoreUnknownValues causes values not matching the schema to be
- // tolerated. Unknown values are ignored. For CSV this ignores extra values
- // at the end of a line. For JSON this ignores named values that do not
- // match any column name. If this field is not set, records containing
- // unknown values are treated as bad records. The MaxBadRecords field can
- // be used to customize how bad records are handled.
- IgnoreUnknownValues bool
-
- // Schema describes the data. It is required when reading CSV or JSON data,
- // unless the data is being loaded into a table that already exists.
- Schema Schema
-
- // Quote is the value used to quote data sections in a CSV file. The
- // default quotation character is the double quote ("), which is used if
- // both Quote and ForceZeroQuote are unset.
- // To specify that no character should be interpreted as a quotation
- // character, set ForceZeroQuote to true.
- // Only used when reading data.
- Quote string
- ForceZeroQuote bool
-}
-
-// quote returns the CSV quote character, or nil if unset.
-func (fc *FileConfig) quote() *string {
- if fc.ForceZeroQuote {
- quote := ""
- return &quote
- }
- if fc.Quote == "" {
- return nil
- }
- return &fc.Quote
-}
-
-func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
- conf.SkipLeadingRows = fc.SkipLeadingRows
- conf.SourceFormat = string(fc.SourceFormat)
- conf.Autodetect = fc.AutoDetect
- conf.AllowJaggedRows = fc.AllowJaggedRows
- conf.AllowQuotedNewlines = fc.AllowQuotedNewlines
- conf.Encoding = string(fc.Encoding)
- conf.FieldDelimiter = fc.FieldDelimiter
- conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
- conf.MaxBadRecords = fc.MaxBadRecords
- if fc.Schema != nil {
- conf.Schema = fc.Schema.asTableSchema()
- }
- conf.Quote = fc.quote()
-}
-
-func (fc *FileConfig) populateExternalDataConfig(conf *bq.ExternalDataConfiguration) {
- format := fc.SourceFormat
- if format == "" {
- // Format must be explicitly set for external data sources.
- format = CSV
- }
- // TODO(jba): support AutoDetect.
- conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
- conf.MaxBadRecords = fc.MaxBadRecords
- conf.SourceFormat = string(format)
- if fc.Schema != nil {
- conf.Schema = fc.Schema.asTableSchema()
- }
- if format == CSV {
- conf.CsvOptions = &bq.CsvOptions{
- AllowJaggedRows: fc.AllowJaggedRows,
- AllowQuotedNewlines: fc.AllowQuotedNewlines,
- Encoding: string(fc.Encoding),
- FieldDelimiter: fc.FieldDelimiter,
- SkipLeadingRows: fc.SkipLeadingRows,
- Quote: fc.quote(),
- }
- }
-}
-
-// DataFormat describes the format of BigQuery table data.
-type DataFormat string
-
-// Constants describing the format of BigQuery table data.
-const (
- CSV DataFormat = "CSV"
- Avro DataFormat = "AVRO"
- JSON DataFormat = "NEWLINE_DELIMITED_JSON"
- DatastoreBackup DataFormat = "DATASTORE_BACKUP"
-)
-
-// Encoding specifies the character encoding of data to be loaded into BigQuery.
-// See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding
-// for more details about how this is used.
-type Encoding string
-
-const (
- UTF_8 Encoding = "UTF-8"
- ISO_8859_1 Encoding = "ISO-8859-1"
-)
diff --git a/vendor/cloud.google.com/go/bigquery/file_test.go b/vendor/cloud.google.com/go/bigquery/file_test.go
deleted file mode 100644
index eb13303ae..000000000
--- a/vendor/cloud.google.com/go/bigquery/file_test.go
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
-
- "cloud.google.com/go/internal/pretty"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func TestQuote(t *testing.T) {
- ptr := func(s string) *string { return &s }
-
- for _, test := range []struct {
- quote string
- force bool
- want *string
- }{
- {"", false, nil},
- {"", true, ptr("")},
- {"-", false, ptr("-")},
- {"-", true, ptr("")},
- } {
- fc := FileConfig{
- Quote: test.quote,
- ForceZeroQuote: test.force,
- }
- got := fc.quote()
- if (got == nil) != (test.want == nil) {
- t.Errorf("%+v\ngot %v\nwant %v", test, pretty.Value(got), pretty.Value(test.want))
- }
- if got != nil && test.want != nil && *got != *test.want {
- t.Errorf("%+v: got %q, want %q", test, *got, *test.want)
- }
- }
-}
-
-func TestPopulateLoadConfig(t *testing.T) {
- hyphen := "-"
- fc := FileConfig{
- SourceFormat: CSV,
- FieldDelimiter: "\t",
- SkipLeadingRows: 8,
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Encoding: UTF_8,
- MaxBadRecords: 7,
- IgnoreUnknownValues: true,
- Schema: Schema{
- stringFieldSchema(),
- nestedFieldSchema(),
- },
- Quote: hyphen,
- }
- want := &bq.JobConfigurationLoad{
- SourceFormat: "CSV",
- FieldDelimiter: "\t",
- SkipLeadingRows: 8,
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Encoding: "UTF-8",
- MaxBadRecords: 7,
- IgnoreUnknownValues: true,
- Schema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqStringFieldSchema(),
- bqNestedFieldSchema(),
- }},
- Quote: &hyphen,
- }
- got := &bq.JobConfigurationLoad{}
- fc.populateLoadConfig(got)
- if !reflect.DeepEqual(got, want) {
- t.Errorf("got:\n%v\nwant:\n%v", pretty.Value(got), pretty.Value(want))
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/gcs.go b/vendor/cloud.google.com/go/bigquery/gcs.go
deleted file mode 100644
index af34b6b5e..000000000
--- a/vendor/cloud.google.com/go/bigquery/gcs.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import bq "google.golang.org/api/bigquery/v2"
-
-// GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute
-// an input or output to a BigQuery operation.
-type GCSReference struct {
- // TODO(jba): Export so that GCSReference can be used to hold data from a Job.get api call and expose it to the user.
- uris []string
-
- FileConfig
-
- // DestinationFormat is the format to use when writing exported files.
- // Allowed values are: CSV, Avro, JSON. The default is CSV.
- // CSV is not supported for tables with nested or repeated fields.
- DestinationFormat DataFormat
-
- // Compression specifies the type of compression to apply when writing data
- // to Google Cloud Storage, or using this GCSReference as an ExternalData
- // source with CSV or JSON SourceFormat. Default is None.
- Compression Compression
-}
-
-// NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination.
-// In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object.
-// Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided.
-// Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name.
-// For more information about the treatment of wildcards and multiple URIs,
-// see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
-func NewGCSReference(uri ...string) *GCSReference {
- return &GCSReference{uris: uri}
-}
-
-// Compression is the type of compression to apply when writing data to Google Cloud Storage.
-type Compression string
-
-const (
- None Compression = "NONE"
- Gzip Compression = "GZIP"
-)
-
-func (gcs *GCSReference) populateInsertJobConfForLoad(conf *insertJobConf) {
- conf.job.Configuration.Load.SourceUris = gcs.uris
- gcs.FileConfig.populateLoadConfig(conf.job.Configuration.Load)
-}
-
-func (gcs *GCSReference) externalDataConfig() bq.ExternalDataConfiguration {
- conf := bq.ExternalDataConfiguration{
- Compression: string(gcs.Compression),
- SourceUris: append([]string{}, gcs.uris...),
- }
- gcs.FileConfig.populateExternalDataConfig(&conf)
- return conf
-}
diff --git a/vendor/cloud.google.com/go/bigquery/integration_test.go b/vendor/cloud.google.com/go/bigquery/integration_test.go
deleted file mode 100644
index 491515281..000000000
--- a/vendor/cloud.google.com/go/bigquery/integration_test.go
+++ /dev/null
@@ -1,1038 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "flag"
- "fmt"
- "log"
- "net/http"
- "os"
- "reflect"
- "sort"
- "strings"
- "testing"
- "time"
-
- gax "github.com/googleapis/gax-go"
-
- "cloud.google.com/go/civil"
- "cloud.google.com/go/internal"
- "cloud.google.com/go/internal/pretty"
- "cloud.google.com/go/internal/testutil"
- "golang.org/x/net/context"
- "google.golang.org/api/googleapi"
- "google.golang.org/api/iterator"
- "google.golang.org/api/option"
-)
-
-var (
- client *Client
- dataset *Dataset
- schema = Schema{
- {Name: "name", Type: StringFieldType},
- {Name: "num", Type: IntegerFieldType},
- }
- testTableExpiration time.Time
-)
-
-func TestMain(m *testing.M) {
- initIntegrationTest()
- os.Exit(m.Run())
-}
-
-func getClient(t *testing.T) *Client {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- return client
-}
-
-// If integration tests will be run, create a unique bucket for them.
-func initIntegrationTest() {
- flag.Parse() // needed for testing.Short()
- if testing.Short() {
- return
- }
- ctx := context.Background()
- ts := testutil.TokenSource(ctx, Scope)
- if ts == nil {
- log.Println("Integration tests skipped. See CONTRIBUTING.md for details")
- return
- }
- projID := testutil.ProjID()
- var err error
- client, err = NewClient(ctx, projID, option.WithTokenSource(ts))
- if err != nil {
- log.Fatalf("NewClient: %v", err)
- }
- dataset = client.Dataset("bigquery_integration_test")
- if err := dataset.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
- log.Fatalf("creating dataset: %v", err)
- }
- testTableExpiration = time.Now().Add(10 * time.Minute).Round(time.Second)
-}
-
-func TestIntegration_Create(t *testing.T) {
- // Check that creating a record field with an empty schema is an error.
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- table := dataset.Table("t_bad")
- schema := Schema{
- {Name: "rec", Type: RecordFieldType, Schema: Schema{}},
- }
- err := table.Create(context.Background(), schema, TableExpiration(time.Now().Add(5*time.Minute)))
- if err == nil {
- t.Fatal("want error, got nil")
- }
- if !hasStatusCode(err, http.StatusBadRequest) {
- t.Fatalf("want a 400 error, got %v", err)
- }
-}
-
-func TestIntegration_CreateView(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- // Test that standard SQL views work.
- view := dataset.Table("t_view_standardsql")
- query := ViewQuery(fmt.Sprintf("SELECT APPROX_COUNT_DISTINCT(name) FROM `%s.%s.%s`", dataset.ProjectID, dataset.DatasetID, table.TableID))
- err := view.Create(context.Background(), UseStandardSQL(), query)
- if err != nil {
- t.Fatalf("table.create: Did not expect an error, got: %v", err)
- }
- view.Delete(ctx)
-}
-
-func TestIntegration_TableMetadata(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
- // Check table metadata.
- md, err := table.Metadata(ctx)
- if err != nil {
- t.Fatal(err)
- }
- // TODO(jba): check md more thorougly.
- if got, want := md.ID, fmt.Sprintf("%s:%s.%s", dataset.ProjectID, dataset.DatasetID, table.TableID); got != want {
- t.Errorf("metadata.ID: got %q, want %q", got, want)
- }
- if got, want := md.Type, RegularTable; got != want {
- t.Errorf("metadata.Type: got %v, want %v", got, want)
- }
- if got, want := md.ExpirationTime, testTableExpiration; !got.Equal(want) {
- t.Errorf("metadata.Type: got %v, want %v", got, want)
- }
-
- // Check that timePartitioning is nil by default
- if md.TimePartitioning != nil {
- t.Errorf("metadata.TimePartitioning: got %v, want %v", md.TimePartitioning, nil)
- }
-
- // Create tables that have time partitioning
- partitionCases := []struct {
- timePartitioning TimePartitioning
- expectedExpiration time.Duration
- }{
- {TimePartitioning{}, time.Duration(0)},
- {TimePartitioning{time.Second}, time.Second},
- }
- for i, c := range partitionCases {
- table := dataset.Table(fmt.Sprintf("t_metadata_partition_%v", i))
- err = table.Create(context.Background(), schema, c.timePartitioning, TableExpiration(time.Now().Add(5*time.Minute)))
- if err != nil {
- t.Fatal(err)
- }
- defer table.Delete(ctx)
- md, err = table.Metadata(ctx)
- if err != nil {
- t.Fatal(err)
- }
-
- got := md.TimePartitioning
- want := &TimePartitioning{c.expectedExpiration}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("metadata.TimePartitioning: got %v, want %v", got, want)
- }
- }
-}
-
-func TestIntegration_DatasetMetadata(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- md, err := dataset.Metadata(ctx)
- if err != nil {
- t.Fatal(err)
- }
- if got, want := md.ID, fmt.Sprintf("%s:%s", dataset.ProjectID, dataset.DatasetID); got != want {
- t.Errorf("ID: got %q, want %q", got, want)
- }
- jan2016 := time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC)
- if md.CreationTime.Before(jan2016) {
- t.Errorf("CreationTime: got %s, want > 2016-1-1", md.CreationTime)
- }
- if md.LastModifiedTime.Before(jan2016) {
- t.Errorf("LastModifiedTime: got %s, want > 2016-1-1", md.LastModifiedTime)
- }
-
- // Verify that we get a NotFound for a nonexistent dataset.
- _, err = client.Dataset("does_not_exist").Metadata(ctx)
- if err == nil || !hasStatusCode(err, http.StatusNotFound) {
- t.Errorf("got %v, want NotFound error", err)
- }
-}
-
-func TestIntegration_DatasetDelete(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- ds := client.Dataset("delete_test")
- if err := ds.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
- t.Fatalf("creating dataset %s: %v", ds, err)
- }
- if err := ds.Delete(ctx); err != nil {
- t.Fatalf("deleting dataset %s: %v", ds, err)
- }
-}
-
-func TestIntegration_Tables(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
- wantName := table.FullyQualifiedName()
-
- // This test is flaky due to eventual consistency.
- ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
- err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
- // Iterate over tables in the dataset.
- it := dataset.Tables(ctx)
- var tableNames []string
- for {
- tbl, err := it.Next()
- if err == iterator.Done {
- break
- }
- if err != nil {
- return false, err
- }
- tableNames = append(tableNames, tbl.FullyQualifiedName())
- }
- // Other tests may be running with this dataset, so there might be more
- // than just our table in the list. So don't try for an exact match; just
- // make sure that our table is there somewhere.
- for _, tn := range tableNames {
- if tn == wantName {
- return true, nil
- }
- }
- return false, fmt.Errorf("got %v\nwant %s in the list", tableNames, wantName)
- })
- if err != nil {
- t.Fatal(err)
- }
-}
-
-func TestIntegration_UploadAndRead(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- // Populate the table.
- upl := table.Uploader()
- var (
- wantRows [][]Value
- saverRows []*ValuesSaver
- )
- for i, name := range []string{"a", "b", "c"} {
- row := []Value{name, int64(i)}
- wantRows = append(wantRows, row)
- saverRows = append(saverRows, &ValuesSaver{
- Schema: schema,
- InsertID: name,
- Row: row,
- })
- }
- if err := upl.Put(ctx, saverRows); err != nil {
- t.Fatal(putError(err))
- }
-
- // Wait until the data has been uploaded. This can take a few seconds, according
- // to https://cloud.google.com/bigquery/streaming-data-into-bigquery.
- if err := waitForRow(ctx, table); err != nil {
- t.Fatal(err)
- }
-
- // Read the table.
- checkRead(t, "upload", table.Read(ctx), wantRows)
-
- // Query the table.
- q := client.Query(fmt.Sprintf("select name, num from %s", table.TableID))
- q.DefaultProjectID = dataset.ProjectID
- q.DefaultDatasetID = dataset.DatasetID
-
- rit, err := q.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- checkRead(t, "query", rit, wantRows)
-
- // Query the long way.
- job1, err := q.Run(ctx)
- if err != nil {
- t.Fatal(err)
- }
-
- job2, err := client.JobFromID(ctx, job1.ID())
- if err != nil {
- t.Fatal(err)
- }
- rit, err = job2.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- checkRead(t, "job.Read", rit, wantRows)
-
- // Get statistics.
- jobStatus, err := job2.Status(ctx)
- if err != nil {
- t.Fatal(err)
- }
- if jobStatus.Statistics == nil {
- t.Fatal("jobStatus missing statistics")
- }
- if _, ok := jobStatus.Statistics.Details.(*QueryStatistics); !ok {
- t.Errorf("expected QueryStatistics, got %T", jobStatus.Statistics.Details)
- }
-
- // Test reading directly into a []Value.
- valueLists, err := readAll(table.Read(ctx))
- if err != nil {
- t.Fatal(err)
- }
- it := table.Read(ctx)
- for i, vl := range valueLists {
- var got []Value
- if err := it.Next(&got); err != nil {
- t.Fatal(err)
- }
- want := []Value(vl)
- if !reflect.DeepEqual(got, want) {
- t.Errorf("%d: got %v, want %v", i, got, want)
- }
- }
-
- // Test reading into a map.
- it = table.Read(ctx)
- for _, vl := range valueLists {
- var vm map[string]Value
- if err := it.Next(&vm); err != nil {
- t.Fatal(err)
- }
- if got, want := len(vm), len(vl); got != want {
- t.Fatalf("valueMap len: got %d, want %d", got, want)
- }
- for i, v := range vl {
- if got, want := vm[schema[i].Name], v; got != want {
- t.Errorf("%d, name=%s: got %v, want %v",
- i, schema[i].Name, got, want)
- }
- }
- }
-}
-
-type SubSubTestStruct struct {
- Integer int64
-}
-
-type SubTestStruct struct {
- String string
- Record SubSubTestStruct
- RecordArray []SubSubTestStruct
-}
-
-type TestStruct struct {
- Name string
- Bytes []byte
- Integer int64
- Float float64
- Boolean bool
- Timestamp time.Time
- Date civil.Date
- Time civil.Time
- DateTime civil.DateTime
-
- StringArray []string
- IntegerArray []int64
- FloatArray []float64
- BooleanArray []bool
- TimestampArray []time.Time
- DateArray []civil.Date
- TimeArray []civil.Time
- DateTimeArray []civil.DateTime
-
- Record SubTestStruct
- RecordArray []SubTestStruct
-}
-
-func TestIntegration_UploadAndReadStructs(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- schema, err := InferSchema(TestStruct{})
- if err != nil {
- t.Fatal(err)
- }
-
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- d := civil.Date{2016, 3, 20}
- tm := civil.Time{15, 4, 5, 0}
- ts := time.Date(2016, 3, 20, 15, 4, 5, 0, time.UTC)
- dtm := civil.DateTime{d, tm}
-
- d2 := civil.Date{1994, 5, 15}
- tm2 := civil.Time{1, 2, 4, 0}
- ts2 := time.Date(1994, 5, 15, 1, 2, 4, 0, time.UTC)
- dtm2 := civil.DateTime{d2, tm2}
-
- // Populate the table.
- upl := table.Uploader()
- want := []*TestStruct{
- {
- "a",
- []byte("byte"),
- 42,
- 3.14,
- true,
- ts,
- d,
- tm,
- dtm,
- []string{"a", "b"},
- []int64{1, 2},
- []float64{1, 1.41},
- []bool{true, false},
- []time.Time{ts, ts2},
- []civil.Date{d, d2},
- []civil.Time{tm, tm2},
- []civil.DateTime{dtm, dtm2},
- SubTestStruct{
- "string",
- SubSubTestStruct{24},
- []SubSubTestStruct{{1}, {2}},
- },
- []SubTestStruct{
- {String: "empty"},
- {
- "full",
- SubSubTestStruct{1},
- []SubSubTestStruct{{1}, {2}},
- },
- },
- },
- {
- Name: "b",
- Bytes: []byte("byte2"),
- Integer: 24,
- Float: 4.13,
- Boolean: false,
- Timestamp: ts,
- Date: d,
- Time: tm,
- DateTime: dtm,
- },
- }
- var savers []*StructSaver
- for _, s := range want {
- savers = append(savers, &StructSaver{Schema: schema, Struct: s})
- }
- if err := upl.Put(ctx, savers); err != nil {
- t.Fatal(putError(err))
- }
-
- // Wait until the data has been uploaded. This can take a few seconds, according
- // to https://cloud.google.com/bigquery/streaming-data-into-bigquery.
- if err := waitForRow(ctx, table); err != nil {
- t.Fatal(err)
- }
-
- // Test iteration with structs.
- it := table.Read(ctx)
- var got []*TestStruct
- for {
- var g TestStruct
- err := it.Next(&g)
- if err == iterator.Done {
- break
- }
- if err != nil {
- t.Fatal(err)
- }
- got = append(got, &g)
- }
- sort.Sort(byName(got))
-
- // BigQuery does not elide nils. It reports an error for nil fields.
- for i, g := range got {
- if i >= len(want) {
- t.Errorf("%d: got %v, past end of want", i, pretty.Value(g))
- } else if w := want[i]; !reflect.DeepEqual(g, w) {
- t.Errorf("%d: got %v, want %v", i, pretty.Value(g), pretty.Value(w))
- }
- }
-}
-
-type byName []*TestStruct
-
-func (b byName) Len() int { return len(b) }
-func (b byName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
-func (b byName) Less(i, j int) bool { return b[i].Name < b[j].Name }
-
-func TestIntegration_Update(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- // Test Update of non-schema fields.
- tm, err := table.Metadata(ctx)
- if err != nil {
- t.Fatal(err)
- }
- wantDescription := tm.Description + "more"
- wantName := tm.Name + "more"
- got, err := table.Update(ctx, TableMetadataToUpdate{
- Description: wantDescription,
- Name: wantName,
- })
- if err != nil {
- t.Fatal(err)
- }
- if got.Description != wantDescription {
- t.Errorf("Description: got %q, want %q", got.Description, wantDescription)
- }
- if got.Name != wantName {
- t.Errorf("Name: got %q, want %q", got.Name, wantName)
- }
- if !reflect.DeepEqual(got.Schema, schema) {
- t.Errorf("Schema: got %v, want %v", pretty.Value(got.Schema), pretty.Value(schema))
- }
-
- // Test schema update.
- // Columns can be added. schema2 is the same as schema, except for the
- // added column in the middle.
- nested := Schema{
- {Name: "nested", Type: BooleanFieldType},
- {Name: "other", Type: StringFieldType},
- }
- schema2 := Schema{
- schema[0],
- {Name: "rec", Type: RecordFieldType, Schema: nested},
- schema[1],
- }
-
- got, err = table.Update(ctx, TableMetadataToUpdate{Schema: schema2})
- if err != nil {
- t.Fatal(err)
- }
-
- // Wherever you add the column, it appears at the end.
- schema3 := Schema{schema2[0], schema2[2], schema2[1]}
- if !reflect.DeepEqual(got.Schema, schema3) {
- t.Errorf("add field:\ngot %v\nwant %v",
- pretty.Value(got.Schema), pretty.Value(schema3))
- }
-
- // Updating with the empty schema succeeds, but is a no-op.
- got, err = table.Update(ctx, TableMetadataToUpdate{Schema: Schema{}})
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(got.Schema, schema3) {
- t.Errorf("empty schema:\ngot %v\nwant %v",
- pretty.Value(got.Schema), pretty.Value(schema3))
- }
-
- // Error cases.
- for _, test := range []struct {
- desc string
- fields []*FieldSchema
- }{
- {"change from optional to required", []*FieldSchema{
- schema3[0],
- {Name: "num", Type: IntegerFieldType, Required: true},
- schema3[2],
- }},
- {"add a required field", []*FieldSchema{
- schema3[0], schema3[1], schema3[2],
- {Name: "req", Type: StringFieldType, Required: true},
- }},
- {"remove a field", []*FieldSchema{schema3[0], schema3[1]}},
- {"remove a nested field", []*FieldSchema{
- schema3[0], schema3[1],
- {Name: "rec", Type: RecordFieldType, Schema: Schema{nested[0]}}}},
- {"remove all nested fields", []*FieldSchema{
- schema3[0], schema3[1],
- {Name: "rec", Type: RecordFieldType, Schema: Schema{}}}},
- } {
- for {
- _, err = table.Update(ctx, TableMetadataToUpdate{Schema: Schema(test.fields)})
- if !hasStatusCode(err, 403) {
- break
- }
- // We've hit the rate limit for updates. Wait a bit and retry.
- t.Logf("%s: retrying after getting %v", test.desc, err)
- time.Sleep(4 * time.Second)
- }
- if err == nil {
- t.Errorf("%s: want error, got nil", test.desc)
- } else if !hasStatusCode(err, 400) {
- t.Errorf("%s: want 400, got %v", test.desc, err)
- }
- }
-}
-
-func TestIntegration_Load(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- // Load the table from a reader.
- r := strings.NewReader("a,0\nb,1\nc,2\n")
- wantRows := [][]Value{
- []Value{"a", int64(0)},
- []Value{"b", int64(1)},
- []Value{"c", int64(2)},
- }
- rs := NewReaderSource(r)
- loader := table.LoaderFrom(rs)
- loader.WriteDisposition = WriteTruncate
- job, err := loader.Run(ctx)
- if err != nil {
- t.Fatal(err)
- }
- if err := wait(ctx, job); err != nil {
- t.Fatal(err)
- }
- checkRead(t, "reader load", table.Read(ctx), wantRows)
-}
-
-func TestIntegration_DML(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- // Retry insert; sometimes it fails with INTERNAL.
- err := internal.Retry(ctx, gax.Backoff{}, func() (bool, error) {
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- // Use DML to insert.
- wantRows := [][]Value{
- []Value{"a", int64(0)},
- []Value{"b", int64(1)},
- []Value{"c", int64(2)},
- }
- query := fmt.Sprintf("INSERT bigquery_integration_test.%s (name, num) "+
- "VALUES ('a', 0), ('b', 1), ('c', 2)",
- table.TableID)
- q := client.Query(query)
- q.UseStandardSQL = true // necessary for DML
- job, err := q.Run(ctx)
- if err != nil {
- return false, err
- }
- if err := wait(ctx, job); err != nil {
- return false, err
- }
- if msg, ok := compareRead(table.Read(ctx), wantRows); !ok {
- // Stop on read error, because that has never been flaky.
- return true, errors.New(msg)
- }
- return true, nil
- })
- if err != nil {
- t.Fatal(err)
- }
-}
-
-func TestIntegration_TimeTypes(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- dtSchema := Schema{
- {Name: "d", Type: DateFieldType},
- {Name: "t", Type: TimeFieldType},
- {Name: "dt", Type: DateTimeFieldType},
- {Name: "ts", Type: TimestampFieldType},
- }
- table := newTable(t, dtSchema)
- defer table.Delete(ctx)
-
- d := civil.Date{2016, 3, 20}
- tm := civil.Time{12, 30, 0, 0}
- ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
- wantRows := [][]Value{
- []Value{d, tm, civil.DateTime{d, tm}, ts},
- }
- upl := table.Uploader()
- if err := upl.Put(ctx, []*ValuesSaver{
- {Schema: dtSchema, Row: wantRows[0]},
- }); err != nil {
- t.Fatal(putError(err))
- }
- if err := waitForRow(ctx, table); err != nil {
- t.Fatal(err)
- }
-
- // SQL wants DATETIMEs with a space between date and time, but the service
- // returns them in RFC3339 form, with a "T" between.
- query := fmt.Sprintf("INSERT bigquery_integration_test.%s (d, t, dt, ts) "+
- "VALUES ('%s', '%s', '%s %s', '%s')",
- table.TableID, d, tm, d, tm, ts.Format("2006-01-02 15:04:05"))
- q := client.Query(query)
- q.UseStandardSQL = true // necessary for DML
- job, err := q.Run(ctx)
- if err != nil {
- t.Fatal(err)
- }
- if err := wait(ctx, job); err != nil {
- t.Fatal(err)
- }
- wantRows = append(wantRows, wantRows[0])
- checkRead(t, "TimeTypes", table.Read(ctx), wantRows)
-}
-
-func TestIntegration_StandardQuery(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
-
- d := civil.Date{2016, 3, 20}
- tm := civil.Time{15, 04, 05, 0}
- ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
- dtm := ts.Format("2006-01-02 15:04:05")
-
- // Constructs Value slices made up of int64s.
- ints := func(args ...int) []Value {
- vals := make([]Value, len(args))
- for i, arg := range args {
- vals[i] = int64(arg)
- }
- return vals
- }
-
- testCases := []struct {
- query string
- wantRow []Value
- }{
- {"SELECT 1", ints(1)},
- {"SELECT 1.3", []Value{1.3}},
- {"SELECT TRUE", []Value{true}},
- {"SELECT 'ABC'", []Value{"ABC"}},
- {"SELECT CAST('foo' AS BYTES)", []Value{[]byte("foo")}},
- {fmt.Sprintf("SELECT TIMESTAMP '%s'", dtm), []Value{ts}},
- {fmt.Sprintf("SELECT [TIMESTAMP '%s', TIMESTAMP '%s']", dtm, dtm), []Value{[]Value{ts, ts}}},
- {fmt.Sprintf("SELECT ('hello', TIMESTAMP '%s')", dtm), []Value{[]Value{"hello", ts}}},
- {fmt.Sprintf("SELECT DATETIME(TIMESTAMP '%s')", dtm), []Value{civil.DateTime{d, tm}}},
- {fmt.Sprintf("SELECT DATE(TIMESTAMP '%s')", dtm), []Value{d}},
- {fmt.Sprintf("SELECT TIME(TIMESTAMP '%s')", dtm), []Value{tm}},
- {"SELECT (1, 2)", []Value{ints(1, 2)}},
- {"SELECT [1, 2, 3]", []Value{ints(1, 2, 3)}},
- {"SELECT ([1, 2], 3, [4, 5])", []Value{[]Value{ints(1, 2), int64(3), ints(4, 5)}}},
- {"SELECT [(1, 2, 3), (4, 5, 6)]", []Value{[]Value{ints(1, 2, 3), ints(4, 5, 6)}}},
- {"SELECT [([1, 2, 3], 4), ([5, 6], 7)]", []Value{[]Value{[]Value{ints(1, 2, 3), int64(4)}, []Value{ints(5, 6), int64(7)}}}},
- {"SELECT ARRAY(SELECT STRUCT([1, 2]))", []Value{[]Value{[]Value{ints(1, 2)}}}},
- }
- for _, c := range testCases {
- q := client.Query(c.query)
- q.UseStandardSQL = true
- it, err := q.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- checkRead(t, "StandardQuery", it, [][]Value{c.wantRow})
- }
-}
-
-func TestIntegration_LegacyQuery(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
-
- ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
- dtm := ts.Format("2006-01-02 15:04:05")
-
- testCases := []struct {
- query string
- wantRow []Value
- }{
- {"SELECT 1", []Value{int64(1)}},
- {"SELECT 1.3", []Value{1.3}},
- {"SELECT TRUE", []Value{true}},
- {"SELECT 'ABC'", []Value{"ABC"}},
- {"SELECT CAST('foo' AS BYTES)", []Value{[]byte("foo")}},
- {fmt.Sprintf("SELECT TIMESTAMP('%s')", dtm), []Value{ts}},
- {fmt.Sprintf("SELECT DATE(TIMESTAMP('%s'))", dtm), []Value{"2016-03-20"}},
- {fmt.Sprintf("SELECT TIME(TIMESTAMP('%s'))", dtm), []Value{"15:04:05"}},
- }
- for _, c := range testCases {
- q := client.Query(c.query)
- it, err := q.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- checkRead(t, "LegacyQuery", it, [][]Value{c.wantRow})
- }
-}
-
-func TestIntegration_QueryParameters(t *testing.T) {
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
-
- d := civil.Date{2016, 3, 20}
- tm := civil.Time{15, 04, 05, 0}
- dtm := civil.DateTime{d, tm}
- ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
-
- type ss struct {
- String string
- }
-
- type s struct {
- Timestamp time.Time
- StringArray []string
- SubStruct ss
- SubStructArray []ss
- }
-
- testCases := []struct {
- query string
- parameters []QueryParameter
- wantRow []Value
- }{
- {"SELECT @val", []QueryParameter{{"val", 1}}, []Value{int64(1)}},
- {"SELECT @val", []QueryParameter{{"val", 1.3}}, []Value{1.3}},
- {"SELECT @val", []QueryParameter{{"val", true}}, []Value{true}},
- {"SELECT @val", []QueryParameter{{"val", "ABC"}}, []Value{"ABC"}},
- {"SELECT @val", []QueryParameter{{"val", []byte("foo")}}, []Value{[]byte("foo")}},
- {"SELECT @val", []QueryParameter{{"val", ts}}, []Value{ts}},
- {"SELECT @val", []QueryParameter{{"val", []time.Time{ts, ts}}}, []Value{[]Value{ts, ts}}},
- {"SELECT @val", []QueryParameter{{"val", dtm}}, []Value{dtm}},
- {"SELECT @val", []QueryParameter{{"val", d}}, []Value{d}},
- {"SELECT @val", []QueryParameter{{"val", tm}}, []Value{tm}},
- {"SELECT @val", []QueryParameter{{"val", s{ts, []string{"a", "b"}, ss{"c"}, []ss{{"d"}, {"e"}}}}},
- []Value{[]Value{ts, []Value{"a", "b"}, []Value{"c"}, []Value{[]Value{"d"}, []Value{"e"}}}}},
- {"SELECT @val.Timestamp, @val.SubStruct.String", []QueryParameter{{"val", s{Timestamp: ts, SubStruct: ss{"a"}}}}, []Value{ts, "a"}},
- }
- for _, c := range testCases {
- q := client.Query(c.query)
- q.Parameters = c.parameters
- it, err := q.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- checkRead(t, "QueryParameters", it, [][]Value{c.wantRow})
- }
-}
-
-func TestIntegration_ReadNullIntoStruct(t *testing.T) {
- // Reading a null into a struct field should return an error (not panic).
- if client == nil {
- t.Skip("Integration tests skipped")
- }
- ctx := context.Background()
- table := newTable(t, schema)
- defer table.Delete(ctx)
-
- upl := table.Uploader()
- row := &ValuesSaver{
- Schema: schema,
- Row: []Value{"name", nil},
- }
- if err := upl.Put(ctx, []*ValuesSaver{row}); err != nil {
- t.Fatal(putError(err))
- }
- if err := waitForRow(ctx, table); err != nil {
- t.Fatal(err)
- }
-
- q := client.Query(fmt.Sprintf("select name, num from %s", table.TableID))
- q.DefaultProjectID = dataset.ProjectID
- q.DefaultDatasetID = dataset.DatasetID
- it, err := q.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- type S struct{ Num int64 }
- var s S
- if err := it.Next(&s); err == nil {
- t.Fatal("got nil, want error")
- }
-}
-
-// Creates a new, temporary table with a unique name and the given schema.
-func newTable(t *testing.T, s Schema) *Table {
- name := fmt.Sprintf("t%d", time.Now().UnixNano())
- table := dataset.Table(name)
- err := table.Create(context.Background(), s, TableExpiration(testTableExpiration))
- if err != nil {
- t.Fatal(err)
- }
- return table
-}
-
-func checkRead(t *testing.T, msg string, it *RowIterator, want [][]Value) {
- if msg2, ok := compareRead(it, want); !ok {
- t.Errorf("%s: %s", msg, msg2)
- }
-}
-
-func compareRead(it *RowIterator, want [][]Value) (msg string, ok bool) {
- got, err := readAll(it)
- if err != nil {
- return err.Error(), false
- }
- if len(got) != len(want) {
- return fmt.Sprintf("got %d rows, want %d", len(got), len(want)), false
- }
- sort.Sort(byCol0(got))
- for i, r := range got {
- gotRow := []Value(r)
- wantRow := want[i]
- if !reflect.DeepEqual(gotRow, wantRow) {
- return fmt.Sprintf("#%d: got %v, want %v", i, gotRow, wantRow), false
- }
- }
- return "", true
-}
-
-func readAll(it *RowIterator) ([][]Value, error) {
- var rows [][]Value
- for {
- var vals []Value
- err := it.Next(&vals)
- if err == iterator.Done {
- return rows, nil
- }
- if err != nil {
- return nil, err
- }
- rows = append(rows, vals)
- }
-}
-
-type byCol0 [][]Value
-
-func (b byCol0) Len() int { return len(b) }
-func (b byCol0) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
-func (b byCol0) Less(i, j int) bool {
- switch a := b[i][0].(type) {
- case string:
- return a < b[j][0].(string)
- case civil.Date:
- return a.Before(b[j][0].(civil.Date))
- default:
- panic("unknown type")
- }
-}
-
-func hasStatusCode(err error, code int) bool {
- if e, ok := err.(*googleapi.Error); ok && e.Code == code {
- return true
- }
- return false
-}
-
-// wait polls the job until it is complete or an error is returned.
-func wait(ctx context.Context, job *Job) error {
- status, err := job.Wait(ctx)
- if err != nil {
- return fmt.Errorf("getting job status: %v", err)
- }
- if status.Err() != nil {
- return fmt.Errorf("job status error: %#v", status.Err())
- }
- if status.Statistics == nil {
- return errors.New("nil Statistics")
- }
- if status.Statistics.EndTime.IsZero() {
- return errors.New("EndTime is zero")
- }
- if status.Statistics.Details == nil {
- return errors.New("nil Statistics.Details")
- }
- return nil
-}
-
-// waitForRow polls the table until it contains a row.
-// TODO(jba): use internal.Retry.
-func waitForRow(ctx context.Context, table *Table) error {
- for {
- it := table.Read(ctx)
- var v []Value
- err := it.Next(&v)
- if err == nil {
- return nil
- }
- if err != iterator.Done {
- return err
- }
- time.Sleep(1 * time.Second)
- }
-}
-
-func putError(err error) string {
- pme, ok := err.(PutMultiError)
- if !ok {
- return err.Error()
- }
- var msgs []string
- for _, err := range pme {
- msgs = append(msgs, err.Error())
- }
- return strings.Join(msgs, "\n")
-}
diff --git a/vendor/cloud.google.com/go/bigquery/iterator.go b/vendor/cloud.google.com/go/bigquery/iterator.go
deleted file mode 100644
index e36149d68..000000000
--- a/vendor/cloud.google.com/go/bigquery/iterator.go
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "fmt"
- "reflect"
-
- "golang.org/x/net/context"
- "google.golang.org/api/iterator"
-)
-
-// A pageFetcher returns a page of rows, starting from the row specified by token.
-type pageFetcher interface {
- fetch(ctx context.Context, s service, token string) (*readDataResult, error)
- setPaging(*pagingConf)
-}
-
-func newRowIterator(ctx context.Context, s service, pf pageFetcher) *RowIterator {
- it := &RowIterator{
- ctx: ctx,
- service: s,
- pf: pf,
- }
- it.pageInfo, it.nextFunc = iterator.NewPageInfo(
- it.fetch,
- func() int { return len(it.rows) },
- func() interface{} { r := it.rows; it.rows = nil; return r })
- return it
-}
-
-// A RowIterator provides access to the result of a BigQuery lookup.
-type RowIterator struct {
- ctx context.Context
- service service
- pf pageFetcher
- pageInfo *iterator.PageInfo
- nextFunc func() error
-
- // StartIndex can be set before the first call to Next. If PageInfo().Token
- // is also set, StartIndex is ignored.
- StartIndex uint64
-
- rows [][]Value
-
- schema Schema // populated on first call to fetch
- structLoader structLoader // used to populate a pointer to a struct
-}
-
-// Next loads the next row into dst. Its return value is iterator.Done if there
-// are no more results. Once Next returns iterator.Done, all subsequent calls
-// will return iterator.Done.
-//
-// dst may implement ValueLoader, or may be a *[]Value, *map[string]Value, or struct pointer.
-//
-// If dst is a *[]Value, it will be set to to new []Value whose i'th element
-// will be populated with the i'th column of the row.
-//
-// If dst is a *map[string]Value, a new map will be created if dst is nil. Then
-// for each schema column name, the map key of that name will be set to the column's
-// value.
-//
-// If dst is pointer to a struct, each column in the schema will be matched
-// with an exported field of the struct that has the same name, ignoring case.
-// Unmatched schema columns and struct fields will be ignored.
-//
-// Each BigQuery column type corresponds to one or more Go types; a matching struct
-// field must be of the correct type. The correspondences are:
-//
-// STRING string
-// BOOL bool
-// INTEGER int, int8, int16, int32, int64, uint8, uint16, uint32
-// FLOAT float32, float64
-// BYTES []byte
-// TIMESTAMP time.Time
-// DATE civil.Date
-// TIME civil.Time
-// DATETIME civil.DateTime
-//
-// A repeated field corresponds to a slice or array of the element type.
-// A RECORD type (nested schema) corresponds to a nested struct or struct pointer.
-// All calls to Next on the same iterator must use the same struct type.
-//
-// It is an error to attempt to read a BigQuery NULL value into a struct field.
-// If your table contains NULLs, use a *[]Value or *map[string]Value.
-func (it *RowIterator) Next(dst interface{}) error {
- var vl ValueLoader
- switch dst := dst.(type) {
- case ValueLoader:
- vl = dst
- case *[]Value:
- vl = (*valueList)(dst)
- case *map[string]Value:
- vl = (*valueMap)(dst)
- default:
- if !isStructPtr(dst) {
- return fmt.Errorf("bigquery: cannot convert %T to ValueLoader (need pointer to []Value, map[string]Value, or struct)", dst)
- }
- }
- if err := it.nextFunc(); err != nil {
- return err
- }
- row := it.rows[0]
- it.rows = it.rows[1:]
-
- if vl == nil {
- // This can only happen if dst is a pointer to a struct. We couldn't
- // set vl above because we need the schema.
- if err := it.structLoader.set(dst, it.schema); err != nil {
- return err
- }
- vl = &it.structLoader
- }
- return vl.Load(row, it.schema)
-}
-
-func isStructPtr(x interface{}) bool {
- t := reflect.TypeOf(x)
- return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
-}
-
-// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
-func (it *RowIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
-
-func (it *RowIterator) fetch(pageSize int, pageToken string) (string, error) {
- pc := &pagingConf{}
- if pageSize > 0 {
- pc.recordsPerRequest = int64(pageSize)
- pc.setRecordsPerRequest = true
- }
- if pageToken == "" {
- pc.startIndex = it.StartIndex
- }
- it.pf.setPaging(pc)
- res, err := it.pf.fetch(it.ctx, it.service, pageToken)
- if err != nil {
- return "", err
- }
- it.rows = append(it.rows, res.rows...)
- it.schema = res.schema
- return res.pageToken, nil
-}
diff --git a/vendor/cloud.google.com/go/bigquery/iterator_test.go b/vendor/cloud.google.com/go/bigquery/iterator_test.go
deleted file mode 100644
index 18bc7294a..000000000
--- a/vendor/cloud.google.com/go/bigquery/iterator_test.go
+++ /dev/null
@@ -1,363 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "fmt"
- "reflect"
- "testing"
-
- "golang.org/x/net/context"
- "google.golang.org/api/iterator"
-)
-
-type fetchResponse struct {
- result *readDataResult // The result to return.
- err error // The error to return.
-}
-
-// pageFetcherStub services fetch requests by returning data from an in-memory list of values.
-type pageFetcherStub struct {
- fetchResponses map[string]fetchResponse
-
- err error
-}
-
-func (pf *pageFetcherStub) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
- call, ok := pf.fetchResponses[token]
- if !ok {
- pf.err = fmt.Errorf("Unexpected page token: %q", token)
- }
- return call.result, call.err
-}
-
-func (pf *pageFetcherStub) setPaging(pc *pagingConf) {}
-
-func TestIterator(t *testing.T) {
- var (
- iiSchema = Schema{
- {Type: IntegerFieldType},
- {Type: IntegerFieldType},
- }
- siSchema = Schema{
- {Type: StringFieldType},
- {Type: IntegerFieldType},
- }
- )
- fetchFailure := errors.New("fetch failure")
-
- testCases := []struct {
- desc string
- pageToken string
- fetchResponses map[string]fetchResponse
- want [][]Value
- wantErr error
- wantSchema Schema
- }{
- {
- desc: "Iteration over single empty page",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{},
- schema: Schema{},
- },
- },
- },
- want: [][]Value{},
- wantSchema: Schema{},
- },
- {
- desc: "Iteration over single page",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- },
- want: [][]Value{{1, 2}, {11, 12}},
- wantSchema: iiSchema,
- },
- {
- desc: "Iteration over single page with different schema",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{"1", 2}, {"11", 12}},
- schema: siSchema,
- },
- },
- },
- want: [][]Value{{"1", 2}, {"11", 12}},
- wantSchema: siSchema,
- },
- {
- desc: "Iteration over two pages",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "a",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- "a": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{101, 102}, {111, 112}},
- schema: iiSchema,
- },
- },
- },
- want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
- wantSchema: iiSchema,
- },
- {
- desc: "Server response includes empty page",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "a",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- "a": {
- result: &readDataResult{
- pageToken: "b",
- rows: [][]Value{},
- schema: iiSchema,
- },
- },
- "b": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{101, 102}, {111, 112}},
- schema: iiSchema,
- },
- },
- },
- want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
- wantSchema: iiSchema,
- },
- {
- desc: "Fetch error",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "a",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- "a": {
- // We returns some data from this fetch, but also an error.
- // So the end result should include only data from the previous fetch.
- err: fetchFailure,
- result: &readDataResult{
- pageToken: "b",
- rows: [][]Value{{101, 102}, {111, 112}},
- schema: iiSchema,
- },
- },
- },
- want: [][]Value{{1, 2}, {11, 12}},
- wantErr: fetchFailure,
- wantSchema: iiSchema,
- },
-
- {
- desc: "Skip over an entire page",
- pageToken: "a",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "a",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- "a": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{101, 102}, {111, 112}},
- schema: iiSchema,
- },
- },
- },
- want: [][]Value{{101, 102}, {111, 112}},
- wantSchema: iiSchema,
- },
-
- {
- desc: "Skip beyond all data",
- pageToken: "b",
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "a",
- rows: [][]Value{{1, 2}, {11, 12}},
- schema: iiSchema,
- },
- },
- "a": {
- result: &readDataResult{
- pageToken: "b",
- rows: [][]Value{{101, 102}, {111, 112}},
- schema: iiSchema,
- },
- },
- "b": {
- result: &readDataResult{},
- },
- },
- // In this test case, Next will return false on its first call,
- // so we won't even attempt to call Get.
- want: [][]Value{},
- wantSchema: Schema{},
- },
- }
-
- for _, tc := range testCases {
- pf := &pageFetcherStub{
- fetchResponses: tc.fetchResponses,
- }
- it := newRowIterator(context.Background(), nil, pf)
- it.PageInfo().Token = tc.pageToken
- values, schema, err := consumeRowIterator(it)
- if err != tc.wantErr {
- t.Fatalf("%s: got %v, want %v", tc.desc, err, tc.wantErr)
- }
- if (len(values) != 0 || len(tc.want) != 0) && !reflect.DeepEqual(values, tc.want) {
- t.Errorf("%s: values:\ngot: %v\nwant:%v", tc.desc, values, tc.want)
- }
- if (len(schema) != 0 || len(tc.wantSchema) != 0) && !reflect.DeepEqual(schema, tc.wantSchema) {
- t.Errorf("%s: iterator.Schema:\ngot: %v\nwant: %v", tc.desc, schema, tc.wantSchema)
- }
- }
-}
-
-type valueListWithSchema struct {
- vals valueList
- schema Schema
-}
-
-func (v *valueListWithSchema) Load(vs []Value, s Schema) error {
- v.vals.Load(vs, s)
- v.schema = s
- return nil
-}
-
-// consumeRowIterator reads the schema and all values from a RowIterator and returns them.
-func consumeRowIterator(it *RowIterator) ([][]Value, Schema, error) {
- var got [][]Value
- var schema Schema
- for {
- var vls valueListWithSchema
- err := it.Next(&vls)
- if err == iterator.Done {
- return got, schema, nil
- }
- if err != nil {
- return got, schema, err
- }
- got = append(got, vls.vals)
- schema = vls.schema
- }
-}
-
-func TestNextDuringErrorState(t *testing.T) {
- pf := &pageFetcherStub{
- fetchResponses: map[string]fetchResponse{
- "": {err: errors.New("bang")},
- },
- }
- it := newRowIterator(context.Background(), nil, pf)
- var vals []Value
- if err := it.Next(&vals); err == nil {
- t.Errorf("Expected error after calling Next")
- }
- if err := it.Next(&vals); err == nil {
- t.Errorf("Expected error calling Next again when iterator has a non-nil error.")
- }
-}
-
-func TestNextAfterFinished(t *testing.T) {
- testCases := []struct {
- fetchResponses map[string]fetchResponse
- want [][]Value
- }{
- {
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{{1, 2}, {11, 12}},
- },
- },
- },
- want: [][]Value{{1, 2}, {11, 12}},
- },
- {
- fetchResponses: map[string]fetchResponse{
- "": {
- result: &readDataResult{
- pageToken: "",
- rows: [][]Value{},
- },
- },
- },
- want: [][]Value{},
- },
- }
-
- for _, tc := range testCases {
- pf := &pageFetcherStub{
- fetchResponses: tc.fetchResponses,
- }
- it := newRowIterator(context.Background(), nil, pf)
-
- values, _, err := consumeRowIterator(it)
- if err != nil {
- t.Fatal(err)
- }
- if (len(values) != 0 || len(tc.want) != 0) && !reflect.DeepEqual(values, tc.want) {
- t.Errorf("values: got:\n%v\nwant:\n%v", values, tc.want)
- }
- // Try calling Get again.
- var vals []Value
- if err := it.Next(&vals); err != iterator.Done {
- t.Errorf("Expected Done calling Next when there are no more values")
- }
- }
-}
-
-func TestIteratorNextTypes(t *testing.T) {
- it := newRowIterator(context.Background(), nil, nil)
- for _, v := range []interface{}{3, "s", []int{}, &[]int{},
- map[string]Value{}, &map[string]interface{}{},
- struct{}{},
- } {
- if err := it.Next(v); err == nil {
- t.Error("%v: want error, got nil", v)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/job.go b/vendor/cloud.google.com/go/bigquery/job.go
deleted file mode 100644
index 89954a2d0..000000000
--- a/vendor/cloud.google.com/go/bigquery/job.go
+++ /dev/null
@@ -1,326 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "time"
-
- "cloud.google.com/go/internal"
- gax "github.com/googleapis/gax-go"
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// A Job represents an operation which has been submitted to BigQuery for processing.
-type Job struct {
- c *Client
- projectID string
- jobID string
-
- isQuery bool
- destinationTable *bq.TableReference // table to read query results from
-}
-
-// JobFromID creates a Job which refers to an existing BigQuery job. The job
-// need not have been created by this package. For example, the job may have
-// been created in the BigQuery console.
-func (c *Client) JobFromID(ctx context.Context, id string) (*Job, error) {
- job, err := c.service.getJob(ctx, c.projectID, id)
- if err != nil {
- return nil, err
- }
- job.c = c
- return job, nil
-}
-
-func (j *Job) ID() string {
- return j.jobID
-}
-
-// State is one of a sequence of states that a Job progresses through as it is processed.
-type State int
-
-const (
- Pending State = iota
- Running
- Done
-)
-
-// JobStatus contains the current State of a job, and errors encountered while processing that job.
-type JobStatus struct {
- State State
-
- err error
-
- // All errors encountered during the running of the job.
- // Not all Errors are fatal, so errors here do not necessarily mean that the job has completed or was unsuccessful.
- Errors []*Error
-
- // Statistics about the job.
- Statistics *JobStatistics
-}
-
-// setJobRef initializes job's JobReference if given a non-empty jobID.
-// projectID must be non-empty.
-func setJobRef(job *bq.Job, jobID, projectID string) {
- if jobID == "" {
- return
- }
- // We don't check whether projectID is empty; the server will return an
- // error when it encounters the resulting JobReference.
-
- job.JobReference = &bq.JobReference{
- JobId: jobID,
- ProjectId: projectID,
- }
-}
-
-// Done reports whether the job has completed.
-// After Done returns true, the Err method will return an error if the job completed unsuccesfully.
-func (s *JobStatus) Done() bool {
- return s.State == Done
-}
-
-// Err returns the error that caused the job to complete unsuccesfully (if any).
-func (s *JobStatus) Err() error {
- return s.err
-}
-
-// Status returns the current status of the job. It fails if the Status could not be determined.
-func (j *Job) Status(ctx context.Context) (*JobStatus, error) {
- js, err := j.c.service.jobStatus(ctx, j.projectID, j.jobID)
- if err != nil {
- return nil, err
- }
- // Fill in the client field of Tables in the statistics.
- if js.Statistics != nil {
- if qs, ok := js.Statistics.Details.(*QueryStatistics); ok {
- for _, t := range qs.ReferencedTables {
- t.c = j.c
- }
- }
- }
- return js, nil
-}
-
-// Cancel requests that a job be cancelled. This method returns without waiting for
-// cancellation to take effect. To check whether the job has terminated, use Job.Status.
-// Cancelled jobs may still incur costs.
-func (j *Job) Cancel(ctx context.Context) error {
- return j.c.service.jobCancel(ctx, j.projectID, j.jobID)
-}
-
-// Wait blocks until the job or the context is done. It returns the final status
-// of the job.
-// If an error occurs while retrieving the status, Wait returns that error. But
-// Wait returns nil if the status was retrieved successfully, even if
-// status.Err() != nil. So callers must check both errors. See the example.
-func (j *Job) Wait(ctx context.Context) (*JobStatus, error) {
- if j.isQuery {
- // We can avoid polling for query jobs.
- if _, err := j.c.service.waitForQuery(ctx, j.projectID, j.jobID); err != nil {
- return nil, err
- }
- // Note: extra RPC even if you just want to wait for the query to finish.
- js, err := j.Status(ctx)
- if err != nil {
- return nil, err
- }
- return js, nil
- }
- // Non-query jobs must poll.
- var js *JobStatus
- err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
- js, err = j.Status(ctx)
- if err != nil {
- return true, err
- }
- if js.Done() {
- return true, nil
- }
- return false, nil
- })
- if err != nil {
- return nil, err
- }
- return js, nil
-}
-
-// Read fetches the results of a query job.
-// If j is not a query job, Read returns an error.
-func (j *Job) Read(ctx context.Context) (*RowIterator, error) {
- if !j.isQuery {
- return nil, errors.New("bigquery: cannot read from a non-query job")
- }
- var projectID string
- if j.destinationTable != nil {
- projectID = j.destinationTable.ProjectId
- } else {
- projectID = j.c.projectID
- }
-
- schema, err := j.c.service.waitForQuery(ctx, projectID, j.jobID)
- if err != nil {
- return nil, err
- }
- // The destination table should only be nil if there was a query error.
- if j.destinationTable == nil {
- return nil, errors.New("bigquery: query job missing destination table")
- }
- return newRowIterator(ctx, j.c.service, &readTableConf{
- projectID: j.destinationTable.ProjectId,
- datasetID: j.destinationTable.DatasetId,
- tableID: j.destinationTable.TableId,
- schema: schema,
- }), nil
-}
-
-// JobStatistics contains statistics about a job.
-type JobStatistics struct {
- CreationTime time.Time
- StartTime time.Time
- EndTime time.Time
- TotalBytesProcessed int64
-
- Details Statistics
-}
-
-// Statistics is one of ExtractStatistics, LoadStatistics or QueryStatistics.
-type Statistics interface {
- implementsStatistics()
-}
-
-// ExtractStatistics contains statistics about an extract job.
-type ExtractStatistics struct {
- // The number of files per destination URI or URI pattern specified in the
- // extract configuration. These values will be in the same order as the
- // URIs specified in the 'destinationUris' field.
- DestinationURIFileCounts []int64
-}
-
-// LoadStatistics contains statistics about a load job.
-type LoadStatistics struct {
- // The number of bytes of source data in a load job.
- InputFileBytes int64
-
- // The number of source files in a load job.
- InputFiles int64
-
- // Size of the loaded data in bytes. Note that while a load job is in the
- // running state, this value may change.
- OutputBytes int64
-
- // The number of rows imported in a load job. Note that while an import job is
- // in the running state, this value may change.
- OutputRows int64
-}
-
-// QueryStatistics contains statistics about a query job.
-type QueryStatistics struct {
- // Billing tier for the job.
- BillingTier int64
-
- // Whether the query result was fetched from the query cache.
- CacheHit bool
-
- // The type of query statement, if valid.
- StatementType string
-
- // Total bytes billed for the job.
- TotalBytesBilled int64
-
- // Total bytes processed for the job.
- TotalBytesProcessed int64
-
- // Describes execution plan for the query.
- QueryPlan []*ExplainQueryStage
-
- // The number of rows affected by a DML statement. Present only for DML
- // statements INSERT, UPDATE or DELETE.
- NumDMLAffectedRows int64
-
- // ReferencedTables: [Output-only, Experimental] Referenced tables for
- // the job. Queries that reference more than 50 tables will not have a
- // complete list.
- ReferencedTables []*Table
-
- // The schema of the results. Present only for successful dry run of
- // non-legacy SQL queries.
- Schema Schema
-
- // Standard SQL: list of undeclared query parameter names detected during a
- // dry run validation.
- UndeclaredQueryParameterNames []string
-}
-
-// ExplainQueryStage describes one stage of a query.
-type ExplainQueryStage struct {
- // Relative amount of the total time the average shard spent on CPU-bound tasks.
- ComputeRatioAvg float64
-
- // Relative amount of the total time the slowest shard spent on CPU-bound tasks.
- ComputeRatioMax float64
-
- // Unique ID for stage within plan.
- ID int64
-
- // Human-readable name for stage.
- Name string
-
- // Relative amount of the total time the average shard spent reading input.
- ReadRatioAvg float64
-
- // Relative amount of the total time the slowest shard spent reading input.
- ReadRatioMax float64
-
- // Number of records read into the stage.
- RecordsRead int64
-
- // Number of records written by the stage.
- RecordsWritten int64
-
- // Current status for the stage.
- Status string
-
- // List of operations within the stage in dependency order (approximately
- // chronological).
- Steps []*ExplainQueryStep
-
- // Relative amount of the total time the average shard spent waiting to be scheduled.
- WaitRatioAvg float64
-
- // Relative amount of the total time the slowest shard spent waiting to be scheduled.
- WaitRatioMax float64
-
- // Relative amount of the total time the average shard spent on writing output.
- WriteRatioAvg float64
-
- // Relative amount of the total time the slowest shard spent on writing output.
- WriteRatioMax float64
-}
-
-// ExplainQueryStep describes one step of a query stage.
-type ExplainQueryStep struct {
- // Machine-readable operation type.
- Kind string
-
- // Human-readable stage descriptions.
- Substeps []string
-}
-
-func (*ExtractStatistics) implementsStatistics() {}
-func (*LoadStatistics) implementsStatistics() {}
-func (*QueryStatistics) implementsStatistics() {}
diff --git a/vendor/cloud.google.com/go/bigquery/load.go b/vendor/cloud.google.com/go/bigquery/load.go
deleted file mode 100644
index 5edf31ca6..000000000
--- a/vendor/cloud.google.com/go/bigquery/load.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// LoadConfig holds the configuration for a load job.
-type LoadConfig struct {
- // JobID is the ID to use for the load job. If unset, a job ID will be automatically created.
- JobID string
-
- // Src is the source from which data will be loaded.
- Src LoadSource
-
- // Dst is the table into which the data will be loaded.
- Dst *Table
-
- // CreateDisposition specifies the circumstances under which the destination table will be created.
- // The default is CreateIfNeeded.
- CreateDisposition TableCreateDisposition
-
- // WriteDisposition specifies how existing data in the destination table is treated.
- // The default is WriteAppend.
- WriteDisposition TableWriteDisposition
-}
-
-// A Loader loads data from Google Cloud Storage into a BigQuery table.
-type Loader struct {
- LoadConfig
- c *Client
-}
-
-// A LoadSource represents a source of data that can be loaded into
-// a BigQuery table.
-//
-// This package defines two LoadSources: GCSReference, for Google Cloud Storage
-// objects, and ReaderSource, for data read from an io.Reader.
-type LoadSource interface {
- populateInsertJobConfForLoad(conf *insertJobConf)
-}
-
-// LoaderFrom returns a Loader which can be used to load data into a BigQuery table.
-// The returned Loader may optionally be further configured before its Run method is called.
-func (t *Table) LoaderFrom(src LoadSource) *Loader {
- return &Loader{
- c: t.c,
- LoadConfig: LoadConfig{
- Src: src,
- Dst: t,
- },
- }
-}
-
-// Run initiates a load job.
-func (l *Loader) Run(ctx context.Context) (*Job, error) {
- job := &bq.Job{
- Configuration: &bq.JobConfiguration{
- Load: &bq.JobConfigurationLoad{
- CreateDisposition: string(l.CreateDisposition),
- WriteDisposition: string(l.WriteDisposition),
- },
- },
- }
- conf := &insertJobConf{job: job}
- l.Src.populateInsertJobConfForLoad(conf)
- setJobRef(job, l.JobID, l.c.projectID)
-
- job.Configuration.Load.DestinationTable = l.Dst.tableRefProto()
-
- return l.c.insertJob(ctx, conf)
-}
diff --git a/vendor/cloud.google.com/go/bigquery/load_test.go b/vendor/cloud.google.com/go/bigquery/load_test.go
deleted file mode 100644
index acb62bc48..000000000
--- a/vendor/cloud.google.com/go/bigquery/load_test.go
+++ /dev/null
@@ -1,229 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "strings"
- "testing"
-
- "golang.org/x/net/context"
-
- "cloud.google.com/go/internal/pretty"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func defaultLoadJob() *bq.Job {
- return &bq.Job{
- Configuration: &bq.JobConfiguration{
- Load: &bq.JobConfigurationLoad{
- DestinationTable: &bq.TableReference{
- ProjectId: "project-id",
- DatasetId: "dataset-id",
- TableId: "table-id",
- },
- SourceUris: []string{"uri"},
- },
- },
- }
-}
-
-func stringFieldSchema() *FieldSchema {
- return &FieldSchema{Name: "fieldname", Type: StringFieldType}
-}
-
-func nestedFieldSchema() *FieldSchema {
- return &FieldSchema{
- Name: "nested",
- Type: RecordFieldType,
- Schema: Schema{stringFieldSchema()},
- }
-}
-
-func bqStringFieldSchema() *bq.TableFieldSchema {
- return &bq.TableFieldSchema{
- Name: "fieldname",
- Type: "STRING",
- }
-}
-
-func bqNestedFieldSchema() *bq.TableFieldSchema {
- return &bq.TableFieldSchema{
- Name: "nested",
- Type: "RECORD",
- Fields: []*bq.TableFieldSchema{bqStringFieldSchema()},
- }
-}
-
-func TestLoad(t *testing.T) {
- c := &Client{projectID: "project-id"}
-
- testCases := []struct {
- dst *Table
- src LoadSource
- config LoadConfig
- want *bq.Job
- }{
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: NewGCSReference("uri"),
- want: defaultLoadJob(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- config: LoadConfig{
- CreateDisposition: CreateNever,
- WriteDisposition: WriteTruncate,
- JobID: "ajob",
- },
- src: NewGCSReference("uri"),
- want: func() *bq.Job {
- j := defaultLoadJob()
- j.Configuration.Load.CreateDisposition = "CREATE_NEVER"
- j.Configuration.Load.WriteDisposition = "WRITE_TRUNCATE"
- j.JobReference = &bq.JobReference{
- JobId: "ajob",
- ProjectId: "project-id",
- }
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: func() *GCSReference {
- g := NewGCSReference("uri")
- g.MaxBadRecords = 1
- g.AllowJaggedRows = true
- g.AllowQuotedNewlines = true
- g.IgnoreUnknownValues = true
- return g
- }(),
- want: func() *bq.Job {
- j := defaultLoadJob()
- j.Configuration.Load.MaxBadRecords = 1
- j.Configuration.Load.AllowJaggedRows = true
- j.Configuration.Load.AllowQuotedNewlines = true
- j.Configuration.Load.IgnoreUnknownValues = true
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: func() *GCSReference {
- g := NewGCSReference("uri")
- g.Schema = Schema{
- stringFieldSchema(),
- nestedFieldSchema(),
- }
- return g
- }(),
- want: func() *bq.Job {
- j := defaultLoadJob()
- j.Configuration.Load.Schema = &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqStringFieldSchema(),
- bqNestedFieldSchema(),
- }}
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: func() *GCSReference {
- g := NewGCSReference("uri")
- g.SkipLeadingRows = 1
- g.SourceFormat = JSON
- g.Encoding = UTF_8
- g.FieldDelimiter = "\t"
- g.Quote = "-"
- return g
- }(),
- want: func() *bq.Job {
- j := defaultLoadJob()
- j.Configuration.Load.SkipLeadingRows = 1
- j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON"
- j.Configuration.Load.Encoding = "UTF-8"
- j.Configuration.Load.FieldDelimiter = "\t"
- hyphen := "-"
- j.Configuration.Load.Quote = &hyphen
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: NewGCSReference("uri"),
- want: func() *bq.Job {
- j := defaultLoadJob()
- // Quote is left unset in GCSReference, so should be nil here.
- j.Configuration.Load.Quote = nil
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: func() *GCSReference {
- g := NewGCSReference("uri")
- g.ForceZeroQuote = true
- return g
- }(),
- want: func() *bq.Job {
- j := defaultLoadJob()
- empty := ""
- j.Configuration.Load.Quote = &empty
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: func() *ReaderSource {
- r := NewReaderSource(strings.NewReader("foo"))
- r.SkipLeadingRows = 1
- r.SourceFormat = JSON
- r.Encoding = UTF_8
- r.FieldDelimiter = "\t"
- r.Quote = "-"
- return r
- }(),
- want: func() *bq.Job {
- j := defaultLoadJob()
- j.Configuration.Load.SourceUris = nil
- j.Configuration.Load.SkipLeadingRows = 1
- j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON"
- j.Configuration.Load.Encoding = "UTF-8"
- j.Configuration.Load.FieldDelimiter = "\t"
- hyphen := "-"
- j.Configuration.Load.Quote = &hyphen
- return j
- }(),
- },
- }
-
- for i, tc := range testCases {
- s := &testService{}
- c.service = s
- loader := tc.dst.LoaderFrom(tc.src)
- tc.config.Src = tc.src
- tc.config.Dst = tc.dst
- loader.LoadConfig = tc.config
- if _, err := loader.Run(context.Background()); err != nil {
- t.Errorf("%d: err calling Loader.Run: %v", i, err)
- continue
- }
- if !reflect.DeepEqual(s.Job, tc.want) {
- t.Errorf("loading %d: got:\n%v\nwant:\n%v",
- i, pretty.Value(s.Job), pretty.Value(tc.want))
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/params.go b/vendor/cloud.google.com/go/bigquery/params.go
deleted file mode 100644
index 2b97f3a2c..000000000
--- a/vendor/cloud.google.com/go/bigquery/params.go
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "encoding/base64"
- "errors"
- "fmt"
- "reflect"
- "regexp"
- "time"
-
- "cloud.google.com/go/civil"
- "cloud.google.com/go/internal/fields"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-var (
- // See https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#timestamp-type.
- timestampFormat = "2006-01-02 15:04:05.999999-07:00"
-
- // See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#schema.fields.name
- validFieldName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]{0,127}$")
-)
-
-func bqTagParser(t reflect.StructTag) (name string, keep bool, other interface{}, err error) {
- if s := t.Get("bigquery"); s != "" {
- if s == "-" {
- return "", false, nil, nil
- }
- if !validFieldName.MatchString(s) {
- return "", false, nil, errInvalidFieldName
- }
- return s, true, nil, nil
- }
- return "", true, nil, nil
-}
-
-var fieldCache = fields.NewCache(bqTagParser, nil, nil)
-
-var (
- int64ParamType = &bq.QueryParameterType{Type: "INT64"}
- float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
- boolParamType = &bq.QueryParameterType{Type: "BOOL"}
- stringParamType = &bq.QueryParameterType{Type: "STRING"}
- bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
- dateParamType = &bq.QueryParameterType{Type: "DATE"}
- timeParamType = &bq.QueryParameterType{Type: "TIME"}
- dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
- timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
-)
-
-var (
- typeOfDate = reflect.TypeOf(civil.Date{})
- typeOfTime = reflect.TypeOf(civil.Time{})
- typeOfDateTime = reflect.TypeOf(civil.DateTime{})
- typeOfGoTime = reflect.TypeOf(time.Time{})
-)
-
-// A QueryParameter is a parameter to a query.
-type QueryParameter struct {
- // Name is used for named parameter mode.
- // It must match the name in the query case-insensitively.
- Name string
-
- // Value is the value of the parameter.
- // The following Go types are supported, with their corresponding
- // Bigquery types:
- // int, int8, int16, int32, int64, uint8, uint16, uint32: INT64
- // Note that uint, uint64 and uintptr are not supported, because
- // they may contain values that cannot fit into a 64-bit signed integer.
- // float32, float64: FLOAT64
- // bool: BOOL
- // string: STRING
- // []byte: BYTES
- // time.Time: TIMESTAMP
- // Arrays and slices of the above.
- // Structs of the above. Only the exported fields are used.
- Value interface{}
-}
-
-func (p QueryParameter) toRaw() (*bq.QueryParameter, error) {
- pv, err := paramValue(reflect.ValueOf(p.Value))
- if err != nil {
- return nil, err
- }
- pt, err := paramType(reflect.TypeOf(p.Value))
- if err != nil {
- return nil, err
- }
- return &bq.QueryParameter{
- Name: p.Name,
- ParameterValue: &pv,
- ParameterType: pt,
- }, nil
-}
-
-func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
- if t == nil {
- return nil, errors.New("bigquery: nil parameter")
- }
- switch t {
- case typeOfDate:
- return dateParamType, nil
- case typeOfTime:
- return timeParamType, nil
- case typeOfDateTime:
- return dateTimeParamType, nil
- case typeOfGoTime:
- return timestampParamType, nil
- }
- switch t.Kind() {
- case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32:
- return int64ParamType, nil
-
- case reflect.Float32, reflect.Float64:
- return float64ParamType, nil
-
- case reflect.Bool:
- return boolParamType, nil
-
- case reflect.String:
- return stringParamType, nil
-
- case reflect.Slice:
- if t.Elem().Kind() == reflect.Uint8 {
- return bytesParamType, nil
- }
- fallthrough
-
- case reflect.Array:
- et, err := paramType(t.Elem())
- if err != nil {
- return nil, err
- }
- return &bq.QueryParameterType{Type: "ARRAY", ArrayType: et}, nil
-
- case reflect.Ptr:
- if t.Elem().Kind() != reflect.Struct {
- break
- }
- t = t.Elem()
- fallthrough
-
- case reflect.Struct:
- var fts []*bq.QueryParameterTypeStructTypes
- fields, err := fieldCache.Fields(t)
- if err != nil {
- return nil, err
- }
- for _, f := range fields {
- pt, err := paramType(f.Type)
- if err != nil {
- return nil, err
- }
- fts = append(fts, &bq.QueryParameterTypeStructTypes{
- Name: f.Name,
- Type: pt,
- })
- }
- return &bq.QueryParameterType{Type: "STRUCT", StructTypes: fts}, nil
- }
- return nil, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter type", t)
-}
-
-func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
- var res bq.QueryParameterValue
- if !v.IsValid() {
- return res, errors.New("bigquery: nil parameter")
- }
- t := v.Type()
- switch t {
- case typeOfDate:
- res.Value = v.Interface().(civil.Date).String()
- return res, nil
-
- case typeOfTime:
- // civil.Time has nanosecond resolution, but BigQuery TIME only microsecond.
- res.Value = civilTimeParamString(v.Interface().(civil.Time))
- return res, nil
-
- case typeOfDateTime:
- dt := v.Interface().(civil.DateTime)
- res.Value = dt.Date.String() + " " + civilTimeParamString(dt.Time)
- return res, nil
-
- case typeOfGoTime:
- res.Value = v.Interface().(time.Time).Format(timestampFormat)
- return res, nil
- }
- switch t.Kind() {
- case reflect.Slice:
- if t.Elem().Kind() == reflect.Uint8 {
- res.Value = base64.StdEncoding.EncodeToString(v.Interface().([]byte))
- return res, nil
- }
- fallthrough
-
- case reflect.Array:
- var vals []*bq.QueryParameterValue
- for i := 0; i < v.Len(); i++ {
- val, err := paramValue(v.Index(i))
- if err != nil {
- return bq.QueryParameterValue{}, err
- }
- vals = append(vals, &val)
- }
- return bq.QueryParameterValue{ArrayValues: vals}, nil
-
- case reflect.Ptr:
- if t.Elem().Kind() != reflect.Struct {
- return res, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter value", t)
- }
- t = t.Elem()
- v = v.Elem()
- if !v.IsValid() {
- // nil pointer becomes empty value
- return res, nil
- }
- fallthrough
-
- case reflect.Struct:
- fields, err := fieldCache.Fields(t)
- if err != nil {
- return bq.QueryParameterValue{}, err
- }
- res.StructValues = map[string]bq.QueryParameterValue{}
- for _, f := range fields {
- fv := v.FieldByIndex(f.Index)
- fp, err := paramValue(fv)
- if err != nil {
- return bq.QueryParameterValue{}, err
- }
- res.StructValues[f.Name] = fp
- }
- return res, nil
- }
- // None of the above: assume a scalar type. (If it's not a valid type,
- // paramType will catch the error.)
- res.Value = fmt.Sprint(v.Interface())
- return res, nil
-}
-
-func civilTimeParamString(t civil.Time) string {
- if t.Nanosecond == 0 {
- return t.String()
- } else {
- micro := (t.Nanosecond + 500) / 1000 // round to nearest microsecond
- t.Nanosecond = 0
- return t.String() + fmt.Sprintf(".%06d", micro)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/params_test.go b/vendor/cloud.google.com/go/bigquery/params_test.go
deleted file mode 100644
index 10da87100..000000000
--- a/vendor/cloud.google.com/go/bigquery/params_test.go
+++ /dev/null
@@ -1,262 +0,0 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "math"
- "reflect"
- "testing"
- "time"
-
- "cloud.google.com/go/civil"
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-var scalarTests = []struct {
- val interface{}
- want string
-}{
- {int64(0), "0"},
- {3.14, "3.14"},
- {3.14159e-87, "3.14159e-87"},
- {true, "true"},
- {"string", "string"},
- {"\u65e5\u672c\u8a9e\n", "\u65e5\u672c\u8a9e\n"},
- {math.NaN(), "NaN"},
- {[]byte("foo"), "Zm9v"}, // base64 encoding of "foo"
- {time.Date(2016, 3, 20, 4, 22, 9, 5000, time.FixedZone("neg1-2", -3720)),
- "2016-03-20 04:22:09.000005-01:02"},
- {civil.Date{2016, 3, 20}, "2016-03-20"},
- {civil.Time{4, 5, 6, 789000000}, "04:05:06.789000"},
- {civil.DateTime{civil.Date{2016, 3, 20}, civil.Time{4, 5, 6, 789000000}}, "2016-03-20 04:05:06.789000"},
-}
-
-type S1 struct {
- A int
- B *S2
- C bool
-}
-
-type S2 struct {
- D string
- e int
-}
-
-var s1 = S1{
- A: 1,
- B: &S2{D: "s"},
- C: true,
-}
-
-func sval(s string) bq.QueryParameterValue {
- return bq.QueryParameterValue{Value: s}
-}
-
-func TestParamValueScalar(t *testing.T) {
- for _, test := range scalarTests {
- got, err := paramValue(reflect.ValueOf(test.val))
- if err != nil {
- t.Errorf("%v: got %v, want nil", test.val, err)
- continue
- }
- want := sval(test.want)
- if !reflect.DeepEqual(got, want) {
- t.Errorf("%v:\ngot %+v\nwant %+v", test.val, got, want)
- }
- }
-}
-
-func TestParamValueArray(t *testing.T) {
- qpv := bq.QueryParameterValue{ArrayValues: []*bq.QueryParameterValue{
- {Value: "1"},
- {Value: "2"},
- },
- }
- for _, test := range []struct {
- val interface{}
- want bq.QueryParameterValue
- }{
- {[]int(nil), bq.QueryParameterValue{}},
- {[]int{}, bq.QueryParameterValue{}},
- {[]int{1, 2}, qpv},
- {[2]int{1, 2}, qpv},
- } {
- got, err := paramValue(reflect.ValueOf(test.val))
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(got, test.want) {
- t.Errorf("%#v:\ngot %+v\nwant %+v", test.val, got, test.want)
- }
- }
-}
-
-func TestParamValueStruct(t *testing.T) {
- got, err := paramValue(reflect.ValueOf(s1))
- if err != nil {
- t.Fatal(err)
- }
- want := bq.QueryParameterValue{
- StructValues: map[string]bq.QueryParameterValue{
- "A": sval("1"),
- "B": bq.QueryParameterValue{
- StructValues: map[string]bq.QueryParameterValue{
- "D": sval("s"),
- },
- },
- "C": sval("true"),
- },
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("got %+v\nwant %+v", got, want)
- }
-}
-
-func TestParamValueErrors(t *testing.T) {
- // paramValue lets a few invalid types through, but paramType catches them.
- // Since we never call one without the other that's fine.
- for _, val := range []interface{}{nil, new([]int)} {
- _, err := paramValue(reflect.ValueOf(val))
- if err == nil {
- t.Errorf("%v (%T): got nil, want error", val, val)
- }
- }
-}
-
-func TestParamType(t *testing.T) {
- for _, test := range []struct {
- val interface{}
- want *bq.QueryParameterType
- }{
- {0, int64ParamType},
- {uint32(32767), int64ParamType},
- {3.14, float64ParamType},
- {float32(3.14), float64ParamType},
- {math.NaN(), float64ParamType},
- {true, boolParamType},
- {"", stringParamType},
- {"string", stringParamType},
- {time.Now(), timestampParamType},
- {[]byte("foo"), bytesParamType},
- {[]int{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: int64ParamType}},
- {[3]bool{}, &bq.QueryParameterType{Type: "ARRAY", ArrayType: boolParamType}},
- {S1{}, &bq.QueryParameterType{
- Type: "STRUCT",
- StructTypes: []*bq.QueryParameterTypeStructTypes{
- {Name: "A", Type: int64ParamType},
- {Name: "B", Type: &bq.QueryParameterType{
- Type: "STRUCT",
- StructTypes: []*bq.QueryParameterTypeStructTypes{
- {Name: "D", Type: stringParamType},
- },
- }},
- {Name: "C", Type: boolParamType},
- },
- }},
- } {
- got, err := paramType(reflect.TypeOf(test.val))
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(got, test.want) {
- t.Errorf("%v (%T): got %v, want %v", test.val, test.val, got, test.want)
- }
- }
-}
-
-func TestParamTypeErrors(t *testing.T) {
- for _, val := range []interface{}{
- nil, uint(0), new([]int), make(chan int),
- } {
- _, err := paramType(reflect.TypeOf(val))
- if err == nil {
- t.Errorf("%v (%T): got nil, want error", val, val)
- }
- }
-}
-
-func TestIntegration_ScalarParam(t *testing.T) {
- c := getClient(t)
- for _, test := range scalarTests {
- got, err := paramRoundTrip(c, test.val)
- if err != nil {
- t.Fatal(err)
- }
- if !equal(got, test.val) {
- t.Errorf("\ngot %#v (%T)\nwant %#v (%T)", got, got, test.val, test.val)
- }
- }
-}
-
-func TestIntegration_OtherParam(t *testing.T) {
- c := getClient(t)
- for _, test := range []struct {
- val interface{}
- want interface{}
- }{
- {[]int(nil), []Value(nil)},
- {[]int{}, []Value(nil)},
- {[]int{1, 2}, []Value{int64(1), int64(2)}},
- {[3]int{1, 2, 3}, []Value{int64(1), int64(2), int64(3)}},
- {S1{}, []Value{int64(0), nil, false}},
- {s1, []Value{int64(1), []Value{"s"}, true}},
- } {
- got, err := paramRoundTrip(c, test.val)
- if err != nil {
- t.Fatal(err)
- }
- if !equal(got, test.want) {
- t.Errorf("\ngot %#v (%T)\nwant %#v (%T)", got, got, test.want, test.want)
- }
- }
-}
-
-func paramRoundTrip(c *Client, x interface{}) (Value, error) {
- q := c.Query("select ?")
- q.Parameters = []QueryParameter{{Value: x}}
- it, err := q.Read(context.Background())
- if err != nil {
- return nil, err
- }
- var val []Value
- err = it.Next(&val)
- if err != nil {
- return nil, err
- }
- if len(val) != 1 {
- return nil, errors.New("wrong number of values")
- }
- return val[0], nil
-}
-
-func equal(x1, x2 interface{}) bool {
- if reflect.TypeOf(x1) != reflect.TypeOf(x2) {
- return false
- }
- switch x1 := x1.(type) {
- case float64:
- if math.IsNaN(x1) {
- return math.IsNaN(x2.(float64))
- }
- return x1 == x2
- case time.Time:
- // BigQuery is only accurate to the microsecond.
- return x1.Round(time.Microsecond).Equal(x2.(time.Time).Round(time.Microsecond))
- default:
- return reflect.DeepEqual(x1, x2)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/query.go b/vendor/cloud.google.com/go/bigquery/query.go
deleted file mode 100644
index d9b63a7df..000000000
--- a/vendor/cloud.google.com/go/bigquery/query.go
+++ /dev/null
@@ -1,206 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// QueryConfig holds the configuration for a query job.
-type QueryConfig struct {
- // JobID is the ID to use for the query job. If this field is empty, a job ID
- // will be automatically created.
- JobID string
-
- // Dst is the table into which the results of the query will be written.
- // If this field is nil, a temporary table will be created.
- Dst *Table
-
- // The query to execute. See https://cloud.google.com/bigquery/query-reference for details.
- Q string
-
- // DefaultProjectID and DefaultDatasetID specify the dataset to use for unqualified table names in the query.
- // If DefaultProjectID is set, DefaultDatasetID must also be set.
- DefaultProjectID string
- DefaultDatasetID string
-
- // TableDefinitions describes data sources outside of BigQuery.
- // The map keys may be used as table names in the query string.
- TableDefinitions map[string]ExternalData
-
- // CreateDisposition specifies the circumstances under which the destination table will be created.
- // The default is CreateIfNeeded.
- CreateDisposition TableCreateDisposition
-
- // WriteDisposition specifies how existing data in the destination table is treated.
- // The default is WriteEmpty.
- WriteDisposition TableWriteDisposition
-
- // DisableQueryCache prevents results being fetched from the query cache.
- // If this field is false, results are fetched from the cache if they are available.
- // The query cache is a best-effort cache that is flushed whenever tables in the query are modified.
- // Cached results are only available when TableID is unspecified in the query's destination Table.
- // For more information, see https://cloud.google.com/bigquery/querying-data#querycaching
- DisableQueryCache bool
-
- // DisableFlattenedResults prevents results being flattened.
- // If this field is false, results from nested and repeated fields are flattened.
- // DisableFlattenedResults implies AllowLargeResults
- // For more information, see https://cloud.google.com/bigquery/docs/data#nested
- DisableFlattenedResults bool
-
- // AllowLargeResults allows the query to produce arbitrarily large result tables.
- // The destination must be a table.
- // When using this option, queries will take longer to execute, even if the result set is small.
- // For additional limitations, see https://cloud.google.com/bigquery/querying-data#largequeryresults
- AllowLargeResults bool
-
- // Priority specifies the priority with which to schedule the query.
- // The default priority is InteractivePriority.
- // For more information, see https://cloud.google.com/bigquery/querying-data#batchqueries
- Priority QueryPriority
-
- // MaxBillingTier sets the maximum billing tier for a Query.
- // Queries that have resource usage beyond this tier will fail (without
- // incurring a charge). If this field is zero, the project default will be used.
- MaxBillingTier int
-
- // MaxBytesBilled limits the number of bytes billed for
- // this job. Queries that would exceed this limit will fail (without incurring
- // a charge).
- // If this field is less than 1, the project default will be
- // used.
- MaxBytesBilled int64
-
- // UseStandardSQL causes the query to use standard SQL.
- // The default is false (using legacy SQL).
- UseStandardSQL bool
-
- // Parameters is a list of query parameters. The presence of parameters
- // implies the use of standard SQL.
- // If the query uses positional syntax ("?"), then no parameter may have a name.
- // If the query uses named syntax ("@p"), then all parameters must have names.
- // It is illegal to mix positional and named syntax.
- Parameters []QueryParameter
-}
-
-// QueryPriority specifies a priority with which a query is to be executed.
-type QueryPriority string
-
-const (
- BatchPriority QueryPriority = "BATCH"
- InteractivePriority QueryPriority = "INTERACTIVE"
-)
-
-// A Query queries data from a BigQuery table. Use Client.Query to create a Query.
-type Query struct {
- client *Client
- QueryConfig
-}
-
-// Query creates a query with string q.
-// The returned Query may optionally be further configured before its Run method is called.
-func (c *Client) Query(q string) *Query {
- return &Query{
- client: c,
- QueryConfig: QueryConfig{Q: q},
- }
-}
-
-// Run initiates a query job.
-func (q *Query) Run(ctx context.Context) (*Job, error) {
- job := &bq.Job{
- Configuration: &bq.JobConfiguration{
- Query: &bq.JobConfigurationQuery{},
- },
- }
- setJobRef(job, q.JobID, q.client.projectID)
-
- if err := q.QueryConfig.populateJobQueryConfig(job.Configuration.Query); err != nil {
- return nil, err
- }
- j, err := q.client.insertJob(ctx, &insertJobConf{job: job})
- if err != nil {
- return nil, err
- }
- j.isQuery = true
- return j, nil
-}
-
-func (q *QueryConfig) populateJobQueryConfig(conf *bq.JobConfigurationQuery) error {
- conf.Query = q.Q
-
- if len(q.TableDefinitions) > 0 {
- conf.TableDefinitions = make(map[string]bq.ExternalDataConfiguration)
- }
- for name, data := range q.TableDefinitions {
- conf.TableDefinitions[name] = data.externalDataConfig()
- }
-
- if q.DefaultProjectID != "" || q.DefaultDatasetID != "" {
- conf.DefaultDataset = &bq.DatasetReference{
- DatasetId: q.DefaultDatasetID,
- ProjectId: q.DefaultProjectID,
- }
- }
-
- if tier := int64(q.MaxBillingTier); tier > 0 {
- conf.MaximumBillingTier = &tier
- }
- conf.CreateDisposition = string(q.CreateDisposition)
- conf.WriteDisposition = string(q.WriteDisposition)
- conf.AllowLargeResults = q.AllowLargeResults
- conf.Priority = string(q.Priority)
-
- f := false
- if q.DisableQueryCache {
- conf.UseQueryCache = &f
- }
- if q.DisableFlattenedResults {
- conf.FlattenResults = &f
- // DisableFlattenResults implies AllowLargeResults.
- conf.AllowLargeResults = true
- }
- if q.MaxBytesBilled >= 1 {
- conf.MaximumBytesBilled = q.MaxBytesBilled
- }
- if q.UseStandardSQL || len(q.Parameters) > 0 {
- conf.UseLegacySql = false
- conf.ForceSendFields = append(conf.ForceSendFields, "UseLegacySql")
- }
-
- if q.Dst != nil && !q.Dst.implicitTable() {
- conf.DestinationTable = q.Dst.tableRefProto()
- }
- for _, p := range q.Parameters {
- qp, err := p.toRaw()
- if err != nil {
- return err
- }
- conf.QueryParameters = append(conf.QueryParameters, qp)
- }
- return nil
-}
-
-// Read submits a query for execution and returns the results via a RowIterator.
-// It is a shorthand for Query.Run followed by Job.Read.
-func (q *Query) Read(ctx context.Context) (*RowIterator, error) {
- job, err := q.Run(ctx)
- if err != nil {
- return nil, err
- }
- return job.Read(ctx)
-}
diff --git a/vendor/cloud.google.com/go/bigquery/query_test.go b/vendor/cloud.google.com/go/bigquery/query_test.go
deleted file mode 100644
index 1715d888e..000000000
--- a/vendor/cloud.google.com/go/bigquery/query_test.go
+++ /dev/null
@@ -1,305 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
-
- "golang.org/x/net/context"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func defaultQueryJob() *bq.Job {
- return &bq.Job{
- Configuration: &bq.JobConfiguration{
- Query: &bq.JobConfigurationQuery{
- DestinationTable: &bq.TableReference{
- ProjectId: "project-id",
- DatasetId: "dataset-id",
- TableId: "table-id",
- },
- Query: "query string",
- DefaultDataset: &bq.DatasetReference{
- ProjectId: "def-project-id",
- DatasetId: "def-dataset-id",
- },
- },
- },
- }
-}
-
-func TestQuery(t *testing.T) {
- c := &Client{
- projectID: "project-id",
- }
- testCases := []struct {
- dst *Table
- src *QueryConfig
- want *bq.Job
- }{
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: defaultQuery,
- want: defaultQueryJob(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.DefaultDataset = nil
- return j
- }(),
- },
- {
- dst: &Table{},
- src: defaultQuery,
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.DestinationTable = nil
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- TableDefinitions: map[string]ExternalData{
- "atable": func() *GCSReference {
- g := NewGCSReference("uri")
- g.AllowJaggedRows = true
- g.AllowQuotedNewlines = true
- g.Compression = Gzip
- g.Encoding = UTF_8
- g.FieldDelimiter = ";"
- g.IgnoreUnknownValues = true
- g.MaxBadRecords = 1
- g.Quote = "'"
- g.SkipLeadingRows = 2
- g.Schema = Schema([]*FieldSchema{
- {Name: "name", Type: StringFieldType},
- })
- return g
- }(),
- },
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.DefaultDataset = nil
- td := make(map[string]bq.ExternalDataConfiguration)
- quote := "'"
- td["atable"] = bq.ExternalDataConfiguration{
- Compression: "GZIP",
- IgnoreUnknownValues: true,
- MaxBadRecords: 1,
- SourceFormat: "CSV", // must be explicitly set.
- SourceUris: []string{"uri"},
- CsvOptions: &bq.CsvOptions{
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Encoding: "UTF-8",
- FieldDelimiter: ";",
- SkipLeadingRows: 2,
- Quote: &quote,
- },
- Schema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- {Name: "name", Type: "STRING"},
- },
- },
- }
- j.Configuration.Query.TableDefinitions = td
- return j
- }(),
- },
- {
- dst: &Table{
- ProjectID: "project-id",
- DatasetID: "dataset-id",
- TableID: "table-id",
- },
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- CreateDisposition: CreateNever,
- WriteDisposition: WriteTruncate,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.WriteDisposition = "WRITE_TRUNCATE"
- j.Configuration.Query.CreateDisposition = "CREATE_NEVER"
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- DisableQueryCache: true,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- f := false
- j.Configuration.Query.UseQueryCache = &f
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- AllowLargeResults: true,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.AllowLargeResults = true
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- DisableFlattenedResults: true,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- f := false
- j.Configuration.Query.FlattenResults = &f
- j.Configuration.Query.AllowLargeResults = true
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- Priority: QueryPriority("low"),
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.Priority = "low"
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- MaxBillingTier: 3,
- MaxBytesBilled: 5,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- tier := int64(3)
- j.Configuration.Query.MaximumBillingTier = &tier
- j.Configuration.Query.MaximumBytesBilled = 5
- return j
- }(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- MaxBytesBilled: -1,
- },
- want: defaultQueryJob(),
- },
- {
- dst: c.Dataset("dataset-id").Table("table-id"),
- src: &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
- UseStandardSQL: true,
- },
- want: func() *bq.Job {
- j := defaultQueryJob()
- j.Configuration.Query.UseLegacySql = false
- j.Configuration.Query.ForceSendFields = []string{"UseLegacySql"}
- return j
- }(),
- },
- }
- for _, tc := range testCases {
- s := &testService{}
- c.service = s
- query := c.Query("")
- query.QueryConfig = *tc.src
- query.Dst = tc.dst
- if _, err := query.Run(context.Background()); err != nil {
- t.Errorf("err calling query: %v", err)
- continue
- }
- if !reflect.DeepEqual(s.Job, tc.want) {
- t.Errorf("querying: got:\n%v\nwant:\n%v", s.Job, tc.want)
- }
- }
-}
-
-func TestConfiguringQuery(t *testing.T) {
- s := &testService{}
- c := &Client{
- projectID: "project-id",
- service: s,
- }
-
- query := c.Query("q")
- query.JobID = "ajob"
- query.DefaultProjectID = "def-project-id"
- query.DefaultDatasetID = "def-dataset-id"
- // Note: Other configuration fields are tested in other tests above.
- // A lot of that can be consolidated once Client.Copy is gone.
-
- want := &bq.Job{
- Configuration: &bq.JobConfiguration{
- Query: &bq.JobConfigurationQuery{
- Query: "q",
- DefaultDataset: &bq.DatasetReference{
- ProjectId: "def-project-id",
- DatasetId: "def-dataset-id",
- },
- },
- },
- JobReference: &bq.JobReference{
- JobId: "ajob",
- ProjectId: "project-id",
- },
- }
-
- if _, err := query.Run(context.Background()); err != nil {
- t.Fatalf("err calling Query.Run: %v", err)
- }
- if !reflect.DeepEqual(s.Job, want) {
- t.Errorf("querying: got:\n%v\nwant:\n%v", s.Job, want)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/read_test.go b/vendor/cloud.google.com/go/bigquery/read_test.go
deleted file mode 100644
index b378d7d2b..000000000
--- a/vendor/cloud.google.com/go/bigquery/read_test.go
+++ /dev/null
@@ -1,260 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "reflect"
- "testing"
-
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
- "google.golang.org/api/iterator"
-)
-
-type readTabledataArgs struct {
- conf *readTableConf
- tok string
-}
-
-// readServiceStub services read requests by returning data from an in-memory list of values.
-type readServiceStub struct {
- // values and pageTokens are used as sources of data to return in response to calls to readTabledata or readQuery.
- values [][][]Value // contains pages / rows / columns.
- pageTokens map[string]string // maps incoming page token to returned page token.
-
- // arguments are recorded for later inspection.
- readTabledataCalls []readTabledataArgs
-
- service
-}
-
-func (s *readServiceStub) readValues(tok string) *readDataResult {
- result := &readDataResult{
- pageToken: s.pageTokens[tok],
- rows: s.values[0],
- }
- s.values = s.values[1:]
-
- return result
-}
-
-func (s *readServiceStub) waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error) {
- return nil, nil
-}
-
-func (s *readServiceStub) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) {
- s.readTabledataCalls = append(s.readTabledataCalls, readTabledataArgs{conf, token})
- return s.readValues(token), nil
-}
-
-func TestRead(t *testing.T) {
- // The data for the service stub to return is populated for each test case in the testCases for loop.
- ctx := context.Background()
- service := &readServiceStub{}
- c := &Client{
- projectID: "project-id",
- service: service,
- }
-
- queryJob := &Job{
- projectID: "project-id",
- jobID: "job-id",
- c: c,
- isQuery: true,
- destinationTable: &bq.TableReference{
- ProjectId: "project-id",
- DatasetId: "dataset-id",
- TableId: "table-id",
- },
- }
-
- for _, readFunc := range []func() *RowIterator{
- func() *RowIterator {
- return c.Dataset("dataset-id").Table("table-id").Read(ctx)
- },
- func() *RowIterator {
- it, err := queryJob.Read(ctx)
- if err != nil {
- t.Fatal(err)
- }
- return it
- },
- } {
- testCases := []struct {
- data [][][]Value
- pageTokens map[string]string
- want [][]Value
- }{
- {
- data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}},
- pageTokens: map[string]string{"": "a", "a": ""},
- want: [][]Value{{1, 2}, {11, 12}, {30, 40}, {31, 41}},
- },
- {
- data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}},
- pageTokens: map[string]string{"": ""}, // no more pages after first one.
- want: [][]Value{{1, 2}, {11, 12}},
- },
- }
- for _, tc := range testCases {
- service.values = tc.data
- service.pageTokens = tc.pageTokens
- if got, ok := collectValues(t, readFunc()); ok {
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("reading: got:\n%v\nwant:\n%v", got, tc.want)
- }
- }
- }
- }
-}
-
-func collectValues(t *testing.T, it *RowIterator) ([][]Value, bool) {
- var got [][]Value
- for {
- var vals []Value
- err := it.Next(&vals)
- if err == iterator.Done {
- break
- }
- if err != nil {
- t.Errorf("err calling Next: %v", err)
- return nil, false
- }
- got = append(got, vals)
- }
- return got, true
-}
-
-func TestNoMoreValues(t *testing.T) {
- c := &Client{
- projectID: "project-id",
- service: &readServiceStub{
- values: [][][]Value{{{1, 2}, {11, 12}}},
- },
- }
- it := c.Dataset("dataset-id").Table("table-id").Read(context.Background())
- var vals []Value
- // We expect to retrieve two values and then fail on the next attempt.
- if err := it.Next(&vals); err != nil {
- t.Fatalf("Next: got: %v: want: nil", err)
- }
- if err := it.Next(&vals); err != nil {
- t.Fatalf("Next: got: %v: want: nil", err)
- }
- if err := it.Next(&vals); err != iterator.Done {
- t.Fatalf("Next: got: %v: want: iterator.Done", err)
- }
-}
-
-type errorReadService struct {
- service
-}
-
-var errBang = errors.New("bang!")
-
-func (s *errorReadService) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) {
- return nil, errBang
-}
-
-func TestReadError(t *testing.T) {
- // test that service read errors are propagated back to the caller.
- c := &Client{
- projectID: "project-id",
- service: &errorReadService{},
- }
- it := c.Dataset("dataset-id").Table("table-id").Read(context.Background())
- var vals []Value
- if err := it.Next(&vals); err != errBang {
- t.Fatalf("Get: got: %v: want: %v", err, errBang)
- }
-}
-
-func TestReadTabledataOptions(t *testing.T) {
- // test that read options are propagated.
- s := &readServiceStub{
- values: [][][]Value{{{1, 2}}},
- }
- c := &Client{
- projectID: "project-id",
- service: s,
- }
- it := c.Dataset("dataset-id").Table("table-id").Read(context.Background())
- it.PageInfo().MaxSize = 5
- var vals []Value
- if err := it.Next(&vals); err != nil {
- t.Fatal(err)
- }
- want := []readTabledataArgs{{
- conf: &readTableConf{
- projectID: "project-id",
- datasetID: "dataset-id",
- tableID: "table-id",
- paging: pagingConf{
- recordsPerRequest: 5,
- setRecordsPerRequest: true,
- },
- },
- tok: "",
- }}
-
- if !reflect.DeepEqual(s.readTabledataCalls, want) {
- t.Errorf("reading: got:\n%v\nwant:\n%v", s.readTabledataCalls, want)
- }
-}
-
-func TestReadQueryOptions(t *testing.T) {
- // test that read options are propagated.
- s := &readServiceStub{
- values: [][][]Value{{{1, 2}}},
- }
- queryJob := &Job{
- projectID: "project-id",
- jobID: "job-id",
- c: &Client{service: s},
- isQuery: true,
- destinationTable: &bq.TableReference{
- ProjectId: "project-id",
- DatasetId: "dataset-id",
- TableId: "table-id",
- },
- }
- it, err := queryJob.Read(context.Background())
- if err != nil {
- t.Fatalf("err calling Read: %v", err)
- }
- it.PageInfo().MaxSize = 5
- var vals []Value
- if err := it.Next(&vals); err != nil {
- t.Fatalf("Next: got: %v: want: nil", err)
- }
-
- want := []readTabledataArgs{{
- conf: &readTableConf{
- projectID: "project-id",
- datasetID: "dataset-id",
- tableID: "table-id",
- paging: pagingConf{
- recordsPerRequest: 5,
- setRecordsPerRequest: true,
- },
- },
- tok: "",
- }}
-
- if !reflect.DeepEqual(s.readTabledataCalls, want) {
- t.Errorf("reading: got:\n%v\nwant:\n%v", s.readTabledataCalls, want)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/schema.go b/vendor/cloud.google.com/go/bigquery/schema.go
deleted file mode 100644
index 0c8f46422..000000000
--- a/vendor/cloud.google.com/go/bigquery/schema.go
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "fmt"
- "reflect"
-
- "cloud.google.com/go/internal/atomiccache"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// Schema describes the fields in a table or query result.
-type Schema []*FieldSchema
-
-type FieldSchema struct {
- // The field name.
- // Must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
- // and must start with a letter or underscore.
- // The maximum length is 128 characters.
- Name string
-
- // A description of the field. The maximum length is 16,384 characters.
- Description string
-
- // Whether the field may contain multiple values.
- Repeated bool
- // Whether the field is required. Ignored if Repeated is true.
- Required bool
-
- // The field data type. If Type is Record, then this field contains a nested schema,
- // which is described by Schema.
- Type FieldType
- // Describes the nested schema if Type is set to Record.
- Schema Schema
-}
-
-func (fs *FieldSchema) asTableFieldSchema() *bq.TableFieldSchema {
- tfs := &bq.TableFieldSchema{
- Description: fs.Description,
- Name: fs.Name,
- Type: string(fs.Type),
- }
-
- if fs.Repeated {
- tfs.Mode = "REPEATED"
- } else if fs.Required {
- tfs.Mode = "REQUIRED"
- } // else leave as default, which is interpreted as NULLABLE.
-
- for _, f := range fs.Schema {
- tfs.Fields = append(tfs.Fields, f.asTableFieldSchema())
- }
-
- return tfs
-}
-
-func (s Schema) asTableSchema() *bq.TableSchema {
- var fields []*bq.TableFieldSchema
- for _, f := range s {
- fields = append(fields, f.asTableFieldSchema())
- }
- return &bq.TableSchema{Fields: fields}
-}
-
-// customizeCreateTable allows a Schema to be used directly as an option to CreateTable.
-func (s Schema) customizeCreateTable(conf *createTableConf) {
- conf.schema = s.asTableSchema()
-}
-
-func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
- fs := &FieldSchema{
- Description: tfs.Description,
- Name: tfs.Name,
- Repeated: tfs.Mode == "REPEATED",
- Required: tfs.Mode == "REQUIRED",
- Type: FieldType(tfs.Type),
- }
-
- for _, f := range tfs.Fields {
- fs.Schema = append(fs.Schema, convertTableFieldSchema(f))
- }
- return fs
-}
-
-func convertTableSchema(ts *bq.TableSchema) Schema {
- if ts == nil {
- return nil
- }
- var s Schema
- for _, f := range ts.Fields {
- s = append(s, convertTableFieldSchema(f))
- }
- return s
-}
-
-type FieldType string
-
-const (
- StringFieldType FieldType = "STRING"
- BytesFieldType FieldType = "BYTES"
- IntegerFieldType FieldType = "INTEGER"
- FloatFieldType FieldType = "FLOAT"
- BooleanFieldType FieldType = "BOOLEAN"
- TimestampFieldType FieldType = "TIMESTAMP"
- RecordFieldType FieldType = "RECORD"
- DateFieldType FieldType = "DATE"
- TimeFieldType FieldType = "TIME"
- DateTimeFieldType FieldType = "DATETIME"
-)
-
-var (
- errNoStruct = errors.New("bigquery: can only infer schema from struct or pointer to struct")
- errUnsupportedFieldType = errors.New("bigquery: unsupported type of field in struct")
- errInvalidFieldName = errors.New("bigquery: invalid name of field in struct")
-)
-
-var typeOfByteSlice = reflect.TypeOf([]byte{})
-
-// InferSchema tries to derive a BigQuery schema from the supplied struct value.
-// NOTE: All fields in the returned Schema are configured to be required,
-// unless the corresponding field in the supplied struct is a slice or array.
-//
-// It is considered an error if the struct (including nested structs) contains
-// any exported fields that are pointers or one of the following types:
-// uint, uint64, uintptr, map, interface, complex64, complex128, func, chan.
-// In these cases, an error will be returned.
-// Future versions may handle these cases without error.
-//
-// Recursively defined structs are also disallowed.
-func InferSchema(st interface{}) (Schema, error) {
- return inferSchemaReflectCached(reflect.TypeOf(st))
-}
-
-var schemaCache atomiccache.Cache
-
-type cacheVal struct {
- schema Schema
- err error
-}
-
-func inferSchemaReflectCached(t reflect.Type) (Schema, error) {
- cv := schemaCache.Get(t, func() interface{} {
- s, err := inferSchemaReflect(t)
- return cacheVal{s, err}
- }).(cacheVal)
- return cv.schema, cv.err
-}
-
-func inferSchemaReflect(t reflect.Type) (Schema, error) {
- rec, err := hasRecursiveType(t, nil)
- if err != nil {
- return nil, err
- }
- if rec {
- return nil, fmt.Errorf("bigquery: schema inference for recursive type %s", t)
- }
- return inferStruct(t)
-}
-
-func inferStruct(t reflect.Type) (Schema, error) {
- switch t.Kind() {
- case reflect.Ptr:
- if t.Elem().Kind() != reflect.Struct {
- return nil, errNoStruct
- }
- t = t.Elem()
- fallthrough
-
- case reflect.Struct:
- return inferFields(t)
- default:
- return nil, errNoStruct
- }
-}
-
-// inferFieldSchema infers the FieldSchema for a Go type
-func inferFieldSchema(rt reflect.Type) (*FieldSchema, error) {
- switch rt {
- case typeOfByteSlice:
- return &FieldSchema{Required: true, Type: BytesFieldType}, nil
- case typeOfGoTime:
- return &FieldSchema{Required: true, Type: TimestampFieldType}, nil
- case typeOfDate:
- return &FieldSchema{Required: true, Type: DateFieldType}, nil
- case typeOfTime:
- return &FieldSchema{Required: true, Type: TimeFieldType}, nil
- case typeOfDateTime:
- return &FieldSchema{Required: true, Type: DateTimeFieldType}, nil
- }
- if isSupportedIntType(rt) {
- return &FieldSchema{Required: true, Type: IntegerFieldType}, nil
- }
- switch rt.Kind() {
- case reflect.Slice, reflect.Array:
- et := rt.Elem()
- if et != typeOfByteSlice && (et.Kind() == reflect.Slice || et.Kind() == reflect.Array) {
- // Multi dimensional slices/arrays are not supported by BigQuery
- return nil, errUnsupportedFieldType
- }
-
- f, err := inferFieldSchema(et)
- if err != nil {
- return nil, err
- }
- f.Repeated = true
- f.Required = false
- return f, nil
- case reflect.Struct, reflect.Ptr:
- nested, err := inferStruct(rt)
- if err != nil {
- return nil, err
- }
- return &FieldSchema{Required: true, Type: RecordFieldType, Schema: nested}, nil
- case reflect.String:
- return &FieldSchema{Required: true, Type: StringFieldType}, nil
- case reflect.Bool:
- return &FieldSchema{Required: true, Type: BooleanFieldType}, nil
- case reflect.Float32, reflect.Float64:
- return &FieldSchema{Required: true, Type: FloatFieldType}, nil
- default:
- return nil, errUnsupportedFieldType
- }
-}
-
-// inferFields extracts all exported field types from struct type.
-func inferFields(rt reflect.Type) (Schema, error) {
- var s Schema
- fields, err := fieldCache.Fields(rt)
- if err != nil {
- return nil, err
- }
- for _, field := range fields {
- f, err := inferFieldSchema(field.Type)
- if err != nil {
- return nil, err
- }
- f.Name = field.Name
- s = append(s, f)
- }
- return s, nil
-}
-
-// isSupportedIntType reports whether t can be properly represented by the
-// BigQuery INTEGER/INT64 type.
-func isSupportedIntType(t reflect.Type) bool {
- switch t.Kind() {
- case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int,
- reflect.Uint8, reflect.Uint16, reflect.Uint32:
- return true
- default:
- return false
- }
-}
-
-// typeList is a linked list of reflect.Types.
-type typeList struct {
- t reflect.Type
- next *typeList
-}
-
-func (l *typeList) has(t reflect.Type) bool {
- for l != nil {
- if l.t == t {
- return true
- }
- l = l.next
- }
- return false
-}
-
-// hasRecursiveType reports whether t or any type inside t refers to itself, directly or indirectly,
-// via exported fields. (Schema inference ignores unexported fields.)
-func hasRecursiveType(t reflect.Type, seen *typeList) (bool, error) {
- for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array {
- t = t.Elem()
- }
- if t.Kind() != reflect.Struct {
- return false, nil
- }
- if seen.has(t) {
- return true, nil
- }
- fields, err := fieldCache.Fields(t)
- if err != nil {
- return false, err
- }
- seen = &typeList{t, seen}
- // Because seen is a linked list, additions to it from one field's
- // recursive call will not affect the value for subsequent fields' calls.
- for _, field := range fields {
- ok, err := hasRecursiveType(field.Type, seen)
- if err != nil {
- return false, err
- }
- if ok {
- return true, nil
- }
- }
- return false, nil
-}
diff --git a/vendor/cloud.google.com/go/bigquery/schema_test.go b/vendor/cloud.google.com/go/bigquery/schema_test.go
deleted file mode 100644
index 9eaaae99f..000000000
--- a/vendor/cloud.google.com/go/bigquery/schema_test.go
+++ /dev/null
@@ -1,796 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "fmt"
- "reflect"
- "testing"
- "time"
-
- "cloud.google.com/go/civil"
- "cloud.google.com/go/internal/pretty"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func (fs *FieldSchema) GoString() string {
- if fs == nil {
- return "<nil>"
- }
-
- return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}",
- fs.Name,
- fs.Description,
- fs.Repeated,
- fs.Required,
- fs.Type,
- fmt.Sprintf("%#v", fs.Schema),
- )
-}
-
-func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema {
- return &bq.TableFieldSchema{
- Description: desc,
- Name: name,
- Mode: mode,
- Type: typ,
- }
-}
-
-func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema {
- return &FieldSchema{
- Description: desc,
- Name: name,
- Repeated: repeated,
- Required: required,
- Type: FieldType(typ),
- }
-}
-
-func TestSchemaConversion(t *testing.T) {
- testCases := []struct {
- schema Schema
- bqSchema *bq.TableSchema
- }{
- {
- // required
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "STRING", false, true),
- },
- },
- {
- // repeated
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "STRING", "REPEATED"),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "STRING", true, false),
- },
- },
- {
- // nullable, string
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "STRING", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "STRING", false, false),
- },
- },
- {
- // integer
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "INTEGER", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "INTEGER", false, false),
- },
- },
- {
- // float
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "FLOAT", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "FLOAT", false, false),
- },
- },
- {
- // boolean
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "BOOLEAN", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "BOOLEAN", false, false),
- },
- },
- {
- // timestamp
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "name", "TIMESTAMP", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "name", "TIMESTAMP", false, false),
- },
- },
- {
- // civil times
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("desc", "f1", "TIME", ""),
- bqTableFieldSchema("desc", "f2", "DATE", ""),
- bqTableFieldSchema("desc", "f3", "DATETIME", ""),
- },
- },
- schema: Schema{
- fieldSchema("desc", "f1", "TIME", false, false),
- fieldSchema("desc", "f2", "DATE", false, false),
- fieldSchema("desc", "f3", "DATETIME", false, false),
- },
- },
- {
- // nested
- bqSchema: &bq.TableSchema{
- Fields: []*bq.TableFieldSchema{
- {
- Description: "An outer schema wrapping a nested schema",
- Name: "outer",
- Mode: "REQUIRED",
- Type: "RECORD",
- Fields: []*bq.TableFieldSchema{
- bqTableFieldSchema("inner field", "inner", "STRING", ""),
- },
- },
- },
- },
- schema: Schema{
- &FieldSchema{
- Description: "An outer schema wrapping a nested schema",
- Name: "outer",
- Required: true,
- Type: "RECORD",
- Schema: []*FieldSchema{
- {
- Description: "inner field",
- Name: "inner",
- Type: "STRING",
- },
- },
- },
- },
- },
- }
-
- for _, tc := range testCases {
- bqSchema := tc.schema.asTableSchema()
- if !reflect.DeepEqual(bqSchema, tc.bqSchema) {
- t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v",
- pretty.Value(bqSchema), pretty.Value(tc.bqSchema))
- }
- schema := convertTableSchema(tc.bqSchema)
- if !reflect.DeepEqual(schema, tc.schema) {
- t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema)
- }
- }
-}
-
-type allStrings struct {
- String string
- ByteSlice []byte
-}
-
-type allSignedIntegers struct {
- Int64 int64
- Int32 int32
- Int16 int16
- Int8 int8
- Int int
-}
-
-type allUnsignedIntegers struct {
- Uint32 uint32
- Uint16 uint16
- Uint8 uint8
-}
-
-type allFloat struct {
- Float64 float64
- Float32 float32
- // NOTE: Complex32 and Complex64 are unsupported by BigQuery
-}
-
-type allBoolean struct {
- Bool bool
-}
-
-type allTime struct {
- Timestamp time.Time
- Time civil.Time
- Date civil.Date
- DateTime civil.DateTime
-}
-
-func reqField(name, typ string) *FieldSchema {
- return &FieldSchema{
- Name: name,
- Type: FieldType(typ),
- Required: true,
- }
-}
-
-func TestSimpleInference(t *testing.T) {
- testCases := []struct {
- in interface{}
- want Schema
- }{
- {
- in: allSignedIntegers{},
- want: Schema{
- reqField("Int64", "INTEGER"),
- reqField("Int32", "INTEGER"),
- reqField("Int16", "INTEGER"),
- reqField("Int8", "INTEGER"),
- reqField("Int", "INTEGER"),
- },
- },
- {
- in: allUnsignedIntegers{},
- want: Schema{
- reqField("Uint32", "INTEGER"),
- reqField("Uint16", "INTEGER"),
- reqField("Uint8", "INTEGER"),
- },
- },
- {
- in: allFloat{},
- want: Schema{
- reqField("Float64", "FLOAT"),
- reqField("Float32", "FLOAT"),
- },
- },
- {
- in: allBoolean{},
- want: Schema{
- reqField("Bool", "BOOLEAN"),
- },
- },
- {
- in: &allBoolean{},
- want: Schema{
- reqField("Bool", "BOOLEAN"),
- },
- },
- {
- in: allTime{},
- want: Schema{
- reqField("Timestamp", "TIMESTAMP"),
- reqField("Time", "TIME"),
- reqField("Date", "DATE"),
- reqField("DateTime", "DATETIME"),
- },
- },
- {
- in: allStrings{},
- want: Schema{
- reqField("String", "STRING"),
- reqField("ByteSlice", "BYTES"),
- },
- },
- }
- for _, tc := range testCases {
- got, err := InferSchema(tc.in)
- if err != nil {
- t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
- }
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in,
- pretty.Value(got), pretty.Value(tc.want))
- }
- }
-}
-
-type containsNested struct {
- hidden string
- NotNested int
- Nested struct {
- Inside int
- }
-}
-
-type containsDoubleNested struct {
- NotNested int
- Nested struct {
- InsideNested struct {
- Inside int
- }
- }
-}
-
-type ptrNested struct {
- Ptr *struct{ Inside int }
-}
-
-type dup struct { // more than one field of the same struct type
- A, B allBoolean
-}
-
-func TestNestedInference(t *testing.T) {
- testCases := []struct {
- in interface{}
- want Schema
- }{
- {
- in: containsNested{},
- want: Schema{
- reqField("NotNested", "INTEGER"),
- &FieldSchema{
- Name: "Nested",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("Inside", "INTEGER")},
- },
- },
- },
- {
- in: containsDoubleNested{},
- want: Schema{
- reqField("NotNested", "INTEGER"),
- &FieldSchema{
- Name: "Nested",
- Required: true,
- Type: "RECORD",
- Schema: Schema{
- {
- Name: "InsideNested",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("Inside", "INTEGER")},
- },
- },
- },
- },
- },
- {
- in: ptrNested{},
- want: Schema{
- &FieldSchema{
- Name: "Ptr",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("Inside", "INTEGER")},
- },
- },
- },
- {
- in: dup{},
- want: Schema{
- &FieldSchema{
- Name: "A",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("Bool", "BOOLEAN")},
- },
- &FieldSchema{
- Name: "B",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("Bool", "BOOLEAN")},
- },
- },
- },
- }
-
- for _, tc := range testCases {
- got, err := InferSchema(tc.in)
- if err != nil {
- t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
- }
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in,
- pretty.Value(got), pretty.Value(tc.want))
- }
- }
-}
-
-type repeated struct {
- NotRepeated []byte
- RepeatedByteSlice [][]byte
- Slice []int
- Array [5]bool
-}
-
-type nestedRepeated struct {
- NotRepeated int
- Repeated []struct {
- Inside int
- }
- RepeatedPtr []*struct{ Inside int }
-}
-
-func repField(name, typ string) *FieldSchema {
- return &FieldSchema{
- Name: name,
- Type: FieldType(typ),
- Repeated: true,
- }
-}
-
-func TestRepeatedInference(t *testing.T) {
- testCases := []struct {
- in interface{}
- want Schema
- }{
- {
- in: repeated{},
- want: Schema{
- reqField("NotRepeated", "BYTES"),
- repField("RepeatedByteSlice", "BYTES"),
- repField("Slice", "INTEGER"),
- repField("Array", "BOOLEAN"),
- },
- },
- {
- in: nestedRepeated{},
- want: Schema{
- reqField("NotRepeated", "INTEGER"),
- {
- Name: "Repeated",
- Repeated: true,
- Type: "RECORD",
- Schema: Schema{reqField("Inside", "INTEGER")},
- },
- {
- Name: "RepeatedPtr",
- Repeated: true,
- Type: "RECORD",
- Schema: Schema{reqField("Inside", "INTEGER")},
- },
- },
- },
- }
-
- for i, tc := range testCases {
- got, err := InferSchema(tc.in)
- if err != nil {
- t.Fatalf("%d: error inferring TableSchema: %v", i, err)
- }
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
- pretty.Value(got), pretty.Value(tc.want))
- }
- }
-}
-
-type Embedded struct {
- Embedded int
-}
-
-type embedded struct {
- Embedded2 int
-}
-
-type nestedEmbedded struct {
- Embedded
- embedded
-}
-
-func TestEmbeddedInference(t *testing.T) {
- got, err := InferSchema(nestedEmbedded{})
- if err != nil {
- t.Fatal(err)
- }
- want := Schema{
- reqField("Embedded", "INTEGER"),
- reqField("Embedded2", "INTEGER"),
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want))
- }
-}
-
-func TestRecursiveInference(t *testing.T) {
- type List struct {
- Val int
- Next *List
- }
-
- _, err := InferSchema(List{})
- if err == nil {
- t.Fatal("got nil, want error")
- }
-}
-
-type withTags struct {
- NoTag int
- ExcludeTag int `bigquery:"-"`
- SimpleTag int `bigquery:"simple_tag"`
- UnderscoreTag int `bigquery:"_id"`
- MixedCase int `bigquery:"MIXEDcase"`
-}
-
-type withTagsNested struct {
- Nested withTags `bigquery:"nested"`
- NestedAnonymous struct {
- ExcludeTag int `bigquery:"-"`
- Inside int `bigquery:"inside"`
- } `bigquery:"anon"`
-}
-
-type withTagsRepeated struct {
- Repeated []withTags `bigquery:"repeated"`
- RepeatedAnonymous []struct {
- ExcludeTag int `bigquery:"-"`
- Inside int `bigquery:"inside"`
- } `bigquery:"anon"`
-}
-
-type withTagsEmbedded struct {
- withTags
-}
-
-var withTagsSchema = Schema{
- reqField("NoTag", "INTEGER"),
- reqField("simple_tag", "INTEGER"),
- reqField("_id", "INTEGER"),
- reqField("MIXEDcase", "INTEGER"),
-}
-
-func TestTagInference(t *testing.T) {
- testCases := []struct {
- in interface{}
- want Schema
- }{
- {
- in: withTags{},
- want: withTagsSchema,
- },
- {
- in: withTagsNested{},
- want: Schema{
- &FieldSchema{
- Name: "nested",
- Required: true,
- Type: "RECORD",
- Schema: withTagsSchema,
- },
- &FieldSchema{
- Name: "anon",
- Required: true,
- Type: "RECORD",
- Schema: Schema{reqField("inside", "INTEGER")},
- },
- },
- },
- {
- in: withTagsRepeated{},
- want: Schema{
- &FieldSchema{
- Name: "repeated",
- Repeated: true,
- Type: "RECORD",
- Schema: withTagsSchema,
- },
- &FieldSchema{
- Name: "anon",
- Repeated: true,
- Type: "RECORD",
- Schema: Schema{reqField("inside", "INTEGER")},
- },
- },
- },
- {
- in: withTagsEmbedded{},
- want: withTagsSchema,
- },
- }
- for i, tc := range testCases {
- got, err := InferSchema(tc.in)
- if err != nil {
- t.Fatalf("%d: error inferring TableSchema: %v", i, err)
- }
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
- pretty.Value(got), pretty.Value(tc.want))
- }
- }
-}
-
-func TestTagInferenceErrors(t *testing.T) {
- testCases := []struct {
- in interface{}
- err error
- }{
- {
- in: struct {
- LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- UnsupporedStartChar int `bigquery:"øab"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- UnsupportedEndChar int `bigquery:"abø"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- UnsupportedMiddleChar int `bigquery:"aøb"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- StartInt int `bigquery:"1abc"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- Hyphens int `bigquery:"a-b"`
- }{},
- err: errInvalidFieldName,
- },
- {
- in: struct {
- OmitEmpty int `bigquery:"abc,omitempty"`
- }{},
- err: errInvalidFieldName,
- },
- }
- for i, tc := range testCases {
- want := tc.err
- _, got := InferSchema(tc.in)
- if !reflect.DeepEqual(got, want) {
- t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i, got, want)
- }
- }
-}
-
-func TestSchemaErrors(t *testing.T) {
- testCases := []struct {
- in interface{}
- err error
- }{
- {
- in: []byte{},
- err: errNoStruct,
- },
- {
- in: new(int),
- err: errNoStruct,
- },
- {
- in: struct{ Uint uint }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Uint64 uint64 }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Uintptr uintptr }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Complex complex64 }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Map map[string]int }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Chan chan bool }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ Ptr *int }{},
- err: errNoStruct,
- },
- {
- in: struct{ Interface interface{} }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ MultiDimensional [][]int }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ MultiDimensional [][][]byte }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ ChanSlice []chan bool }{},
- err: errUnsupportedFieldType,
- },
- {
- in: struct{ NestedChan struct{ Chan []chan bool } }{},
- err: errUnsupportedFieldType,
- },
- }
- for _, tc := range testCases {
- want := tc.err
- _, got := InferSchema(tc.in)
- if !reflect.DeepEqual(got, want) {
- t.Errorf("%#v: got:\n%#v\nwant:\n%#v", tc.in, got, want)
- }
- }
-}
-
-func TestHasRecursiveType(t *testing.T) {
- type (
- nonStruct int
- nonRec struct{ A string }
- dup struct{ A, B nonRec }
- rec struct {
- A int
- B *rec
- }
- recUnexported struct {
- A int
- b *rec
- }
- hasRec struct {
- A int
- R *rec
- }
- recSlicePointer struct {
- A []*recSlicePointer
- }
- )
- for _, test := range []struct {
- in interface{}
- want bool
- }{
- {nonStruct(0), false},
- {nonRec{}, false},
- {dup{}, false},
- {rec{}, true},
- {recUnexported{}, false},
- {hasRec{}, true},
- {&recSlicePointer{}, true},
- } {
- got, err := hasRecursiveType(reflect.TypeOf(test.in), nil)
- if err != nil {
- t.Fatal(err)
- }
- if got != test.want {
- t.Errorf("%T: got %t, want %t", test.in, got, test.want)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/service.go b/vendor/cloud.google.com/go/bigquery/service.go
deleted file mode 100644
index cadd7b4c2..000000000
--- a/vendor/cloud.google.com/go/bigquery/service.go
+++ /dev/null
@@ -1,731 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "fmt"
- "io"
- "net/http"
- "sync"
- "time"
-
- "cloud.google.com/go/internal"
- "cloud.google.com/go/internal/version"
- gax "github.com/googleapis/gax-go"
-
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
- "google.golang.org/api/googleapi"
-)
-
-// service provides an internal abstraction to isolate the generated
-// BigQuery API; most of this package uses this interface instead.
-// The single implementation, *bigqueryService, contains all the knowledge
-// of the generated BigQuery API.
-type service interface {
- // Jobs
- insertJob(ctx context.Context, projectId string, conf *insertJobConf) (*Job, error)
- getJob(ctx context.Context, projectId, jobID string) (*Job, error)
- jobCancel(ctx context.Context, projectId, jobID string) error
- jobStatus(ctx context.Context, projectId, jobID string) (*JobStatus, error)
-
- // Tables
- createTable(ctx context.Context, conf *createTableConf) error
- getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error)
- deleteTable(ctx context.Context, projectID, datasetID, tableID string) error
-
- // listTables returns a page of Tables and a next page token. Note: the Tables do not have their c field populated.
- listTables(ctx context.Context, projectID, datasetID string, pageSize int, pageToken string) ([]*Table, string, error)
- patchTable(ctx context.Context, projectID, datasetID, tableID string, conf *patchTableConf) (*TableMetadata, error)
-
- // Table data
- readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error)
- insertRows(ctx context.Context, projectID, datasetID, tableID string, rows []*insertionRow, conf *insertRowsConf) error
-
- // Datasets
- insertDataset(ctx context.Context, datasetID, projectID string) error
- deleteDataset(ctx context.Context, datasetID, projectID string) error
- getDatasetMetadata(ctx context.Context, projectID, datasetID string) (*DatasetMetadata, error)
-
- // Misc
-
- // Waits for a query to complete.
- waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error)
-
- // listDatasets returns a page of Datasets and a next page token. Note: the Datasets do not have their c field populated.
- listDatasets(ctx context.Context, projectID string, maxResults int, pageToken string, all bool, filter string) ([]*Dataset, string, error)
-}
-
-var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
-
-func setClientHeader(headers http.Header) {
- headers.Set("x-goog-api-client", xGoogHeader)
-}
-
-type bigqueryService struct {
- s *bq.Service
-}
-
-func newBigqueryService(client *http.Client, endpoint string) (*bigqueryService, error) {
- s, err := bq.New(client)
- if err != nil {
- return nil, fmt.Errorf("constructing bigquery client: %v", err)
- }
- s.BasePath = endpoint
-
- return &bigqueryService{s: s}, nil
-}
-
-// getPages calls the supplied getPage function repeatedly until there are no pages left to get.
-// token is the token of the initial page to start from. Use an empty string to start from the beginning.
-func getPages(token string, getPage func(token string) (nextToken string, err error)) error {
- for {
- var err error
- token, err = getPage(token)
- if err != nil {
- return err
- }
- if token == "" {
- return nil
- }
- }
-}
-
-type insertJobConf struct {
- job *bq.Job
- media io.Reader
-}
-
-// Calls the Jobs.Insert RPC and returns a Job. Callers must set the returned Job's
-// client.
-func (s *bigqueryService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
- call := s.s.Jobs.Insert(projectID, conf.job).Context(ctx)
- setClientHeader(call.Header())
- if conf.media != nil {
- call.Media(conf.media)
- }
- var res *bq.Job
- var err error
- invoke := func() error {
- res, err = call.Do()
- return err
- }
- // A job with a client-generated ID can be retried; the presence of the
- // ID makes the insert operation idempotent.
- // We don't retry if there is media, because it is an io.Reader. We'd
- // have to read the contents and keep it in memory, and that could be expensive.
- // TODO(jba): Look into retrying if media != nil.
- if conf.job.JobReference != nil && conf.media == nil {
- err = runWithRetry(ctx, invoke)
- } else {
- err = invoke()
- }
- if err != nil {
- return nil, err
- }
-
- var dt *bq.TableReference
- if qc := res.Configuration.Query; qc != nil {
- dt = qc.DestinationTable
- }
- return &Job{
- projectID: projectID,
- jobID: res.JobReference.JobId,
- destinationTable: dt,
- }, nil
-}
-
-type pagingConf struct {
- recordsPerRequest int64
- setRecordsPerRequest bool
-
- startIndex uint64
-}
-
-type readTableConf struct {
- projectID, datasetID, tableID string
- paging pagingConf
- schema Schema // lazily initialized when the first page of data is fetched.
-}
-
-func (conf *readTableConf) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
- return s.readTabledata(ctx, conf, token)
-}
-
-func (conf *readTableConf) setPaging(pc *pagingConf) { conf.paging = *pc }
-
-type readDataResult struct {
- pageToken string
- rows [][]Value
- totalRows uint64
- schema Schema
-}
-
-func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error) {
- // Prepare request to fetch one page of table data.
- req := s.s.Tabledata.List(conf.projectID, conf.datasetID, conf.tableID)
- setClientHeader(req.Header())
-
- if pageToken != "" {
- req.PageToken(pageToken)
- } else {
- req.StartIndex(conf.paging.startIndex)
- }
-
- if conf.paging.setRecordsPerRequest {
- req.MaxResults(conf.paging.recordsPerRequest)
- }
-
- // Fetch the table schema in the background, if necessary.
- var schemaErr error
- var schemaFetch sync.WaitGroup
- if conf.schema == nil {
- schemaFetch.Add(1)
- go func() {
- defer schemaFetch.Done()
- var t *bq.Table
- t, schemaErr = s.s.Tables.Get(conf.projectID, conf.datasetID, conf.tableID).
- Fields("schema").
- Context(ctx).
- Do()
- if schemaErr == nil && t.Schema != nil {
- conf.schema = convertTableSchema(t.Schema)
- }
- }()
- }
-
- res, err := req.Context(ctx).Do()
- if err != nil {
- return nil, err
- }
-
- schemaFetch.Wait()
- if schemaErr != nil {
- return nil, schemaErr
- }
-
- result := &readDataResult{
- pageToken: res.PageToken,
- totalRows: uint64(res.TotalRows),
- schema: conf.schema,
- }
- result.rows, err = convertRows(res.Rows, conf.schema)
- if err != nil {
- return nil, err
- }
- return result, nil
-}
-
-func (s *bigqueryService) waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error) {
- // Use GetQueryResults only to wait for completion, not to read results.
- req := s.s.Jobs.GetQueryResults(projectID, jobID).Context(ctx).MaxResults(0)
- setClientHeader(req.Header())
- backoff := gax.Backoff{
- Initial: 1 * time.Second,
- Multiplier: 2,
- Max: 60 * time.Second,
- }
- var res *bq.GetQueryResultsResponse
- err := internal.Retry(ctx, backoff, func() (stop bool, err error) {
- res, err = req.Do()
- if err != nil {
- return !retryableError(err), err
- }
- if !res.JobComplete { // GetQueryResults may return early without error; retry.
- return false, nil
- }
- return true, nil
- })
- if err != nil {
- return nil, err
- }
- return convertTableSchema(res.Schema), nil
-}
-
-type insertRowsConf struct {
- templateSuffix string
- ignoreUnknownValues bool
- skipInvalidRows bool
-}
-
-func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID, tableID string, rows []*insertionRow, conf *insertRowsConf) error {
- req := &bq.TableDataInsertAllRequest{
- TemplateSuffix: conf.templateSuffix,
- IgnoreUnknownValues: conf.ignoreUnknownValues,
- SkipInvalidRows: conf.skipInvalidRows,
- }
- for _, row := range rows {
- m := make(map[string]bq.JsonValue)
- for k, v := range row.Row {
- m[k] = bq.JsonValue(v)
- }
- req.Rows = append(req.Rows, &bq.TableDataInsertAllRequestRows{
- InsertId: row.InsertID,
- Json: m,
- })
- }
- var res *bq.TableDataInsertAllResponse
- err := runWithRetry(ctx, func() error {
- var err error
- req := s.s.Tabledata.InsertAll(projectID, datasetID, tableID, req).Context(ctx)
- setClientHeader(req.Header())
- res, err = req.Do()
- return err
- })
- if err != nil {
- return err
- }
- if len(res.InsertErrors) == 0 {
- return nil
- }
-
- var errs PutMultiError
- for _, e := range res.InsertErrors {
- if int(e.Index) > len(rows) {
- return fmt.Errorf("internal error: unexpected row index: %v", e.Index)
- }
- rie := RowInsertionError{
- InsertID: rows[e.Index].InsertID,
- RowIndex: int(e.Index),
- }
- for _, errp := range e.Errors {
- rie.Errors = append(rie.Errors, errorFromErrorProto(errp))
- }
- errs = append(errs, rie)
- }
- return errs
-}
-
-func (s *bigqueryService) getJob(ctx context.Context, projectID, jobID string) (*Job, error) {
- res, err := s.s.Jobs.Get(projectID, jobID).
- Fields("configuration").
- Context(ctx).
- Do()
- if err != nil {
- return nil, err
- }
- var isQuery bool
- var dest *bq.TableReference
- if res.Configuration.Query != nil {
- isQuery = true
- dest = res.Configuration.Query.DestinationTable
- }
- return &Job{
- projectID: projectID,
- jobID: jobID,
- isQuery: isQuery,
- destinationTable: dest,
- }, nil
-}
-
-func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string) error {
- // Jobs.Cancel returns a job entity, but the only relevant piece of
- // data it may contain (the status of the job) is unreliable. From the
- // docs: "This call will return immediately, and the client will need
- // to poll for the job status to see if the cancel completed
- // successfully". So it would be misleading to return a status.
- _, err := s.s.Jobs.Cancel(projectID, jobID).
- Fields(). // We don't need any of the response data.
- Context(ctx).
- Do()
- return err
-}
-
-func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
- res, err := s.s.Jobs.Get(projectID, jobID).
- Fields("status", "statistics"). // Only fetch what we need.
- Context(ctx).
- Do()
- if err != nil {
- return nil, err
- }
- st, err := jobStatusFromProto(res.Status)
- if err != nil {
- return nil, err
- }
- st.Statistics = jobStatisticsFromProto(res.Statistics)
- return st, nil
-}
-
-var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done}
-
-func jobStatusFromProto(status *bq.JobStatus) (*JobStatus, error) {
- state, ok := stateMap[status.State]
- if !ok {
- return nil, fmt.Errorf("unexpected job state: %v", status.State)
- }
-
- newStatus := &JobStatus{
- State: state,
- err: nil,
- }
- if err := errorFromErrorProto(status.ErrorResult); state == Done && err != nil {
- newStatus.err = err
- }
-
- for _, ep := range status.Errors {
- newStatus.Errors = append(newStatus.Errors, errorFromErrorProto(ep))
- }
- return newStatus, nil
-}
-
-func jobStatisticsFromProto(s *bq.JobStatistics) *JobStatistics {
- js := &JobStatistics{
- CreationTime: unixMillisToTime(s.CreationTime),
- StartTime: unixMillisToTime(s.StartTime),
- EndTime: unixMillisToTime(s.EndTime),
- TotalBytesProcessed: s.TotalBytesProcessed,
- }
- switch {
- case s.Extract != nil:
- js.Details = &ExtractStatistics{
- DestinationURIFileCounts: []int64(s.Extract.DestinationUriFileCounts),
- }
- case s.Load != nil:
- js.Details = &LoadStatistics{
- InputFileBytes: s.Load.InputFileBytes,
- InputFiles: s.Load.InputFiles,
- OutputBytes: s.Load.OutputBytes,
- OutputRows: s.Load.OutputRows,
- }
- case s.Query != nil:
- var names []string
- for _, qp := range s.Query.UndeclaredQueryParameters {
- names = append(names, qp.Name)
- }
- var tables []*Table
- for _, tr := range s.Query.ReferencedTables {
- tables = append(tables, convertTableReference(tr))
- }
- js.Details = &QueryStatistics{
- BillingTier: s.Query.BillingTier,
- CacheHit: s.Query.CacheHit,
- StatementType: s.Query.StatementType,
- TotalBytesBilled: s.Query.TotalBytesBilled,
- TotalBytesProcessed: s.Query.TotalBytesProcessed,
- NumDMLAffectedRows: s.Query.NumDmlAffectedRows,
- QueryPlan: queryPlanFromProto(s.Query.QueryPlan),
- Schema: convertTableSchema(s.Query.Schema),
- ReferencedTables: tables,
- UndeclaredQueryParameterNames: names,
- }
- }
- return js
-}
-
-func queryPlanFromProto(stages []*bq.ExplainQueryStage) []*ExplainQueryStage {
- var res []*ExplainQueryStage
- for _, s := range stages {
- var steps []*ExplainQueryStep
- for _, p := range s.Steps {
- steps = append(steps, &ExplainQueryStep{
- Kind: p.Kind,
- Substeps: p.Substeps,
- })
- }
- res = append(res, &ExplainQueryStage{
- ComputeRatioAvg: s.ComputeRatioAvg,
- ComputeRatioMax: s.ComputeRatioMax,
- ID: s.Id,
- Name: s.Name,
- ReadRatioAvg: s.ReadRatioAvg,
- ReadRatioMax: s.ReadRatioMax,
- RecordsRead: s.RecordsRead,
- RecordsWritten: s.RecordsWritten,
- Status: s.Status,
- Steps: steps,
- WaitRatioAvg: s.WaitRatioAvg,
- WaitRatioMax: s.WaitRatioMax,
- WriteRatioAvg: s.WriteRatioAvg,
- WriteRatioMax: s.WriteRatioMax,
- })
- }
- return res
-}
-
-// listTables returns a subset of tables that belong to a dataset, and a token for fetching the next subset.
-func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID string, pageSize int, pageToken string) ([]*Table, string, error) {
- var tables []*Table
- req := s.s.Tables.List(projectID, datasetID).
- PageToken(pageToken).
- Context(ctx)
- setClientHeader(req.Header())
- if pageSize > 0 {
- req.MaxResults(int64(pageSize))
- }
- res, err := req.Do()
- if err != nil {
- return nil, "", err
- }
- for _, t := range res.Tables {
- tables = append(tables, convertTableReference(t.TableReference))
- }
- return tables, res.NextPageToken, nil
-}
-
-type createTableConf struct {
- projectID, datasetID, tableID string
- expiration time.Time
- viewQuery string
- schema *bq.TableSchema
- useStandardSQL bool
- timePartitioning *TimePartitioning
-}
-
-// createTable creates a table in the BigQuery service.
-// expiration is an optional time after which the table will be deleted and its storage reclaimed.
-// If viewQuery is non-empty, the created table will be of type VIEW.
-// Note: expiration can only be set during table creation.
-// Note: after table creation, a view can be modified only if its table was initially created with a view.
-func (s *bigqueryService) createTable(ctx context.Context, conf *createTableConf) error {
- table := &bq.Table{
- TableReference: &bq.TableReference{
- ProjectId: conf.projectID,
- DatasetId: conf.datasetID,
- TableId: conf.tableID,
- },
- }
- if !conf.expiration.IsZero() {
- table.ExpirationTime = conf.expiration.UnixNano() / 1e6
- }
- // TODO(jba): make it impossible to provide both a view query and a schema.
- if conf.viewQuery != "" {
- table.View = &bq.ViewDefinition{
- Query: conf.viewQuery,
- }
- if conf.useStandardSQL {
- table.View.UseLegacySql = false
- table.View.ForceSendFields = append(table.View.ForceSendFields, "UseLegacySql")
- }
- }
- if conf.schema != nil {
- table.Schema = conf.schema
- }
- if conf.timePartitioning != nil {
- table.TimePartitioning = &bq.TimePartitioning{
- Type: "DAY",
- ExpirationMs: int64(conf.timePartitioning.Expiration.Seconds() * 1000),
- }
- }
-
- req := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx)
- setClientHeader(req.Header())
- _, err := req.Do()
- return err
-}
-
-func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) {
- req := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx)
- setClientHeader(req.Header())
- table, err := req.Do()
- if err != nil {
- return nil, err
- }
- return bqTableToMetadata(table), nil
-}
-
-func (s *bigqueryService) deleteTable(ctx context.Context, projectID, datasetID, tableID string) error {
- req := s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx)
- setClientHeader(req.Header())
- return req.Do()
-}
-
-func bqTableToMetadata(t *bq.Table) *TableMetadata {
- md := &TableMetadata{
- Description: t.Description,
- Name: t.FriendlyName,
- Type: TableType(t.Type),
- ID: t.Id,
- NumBytes: t.NumBytes,
- NumRows: t.NumRows,
- ExpirationTime: unixMillisToTime(t.ExpirationTime),
- CreationTime: unixMillisToTime(t.CreationTime),
- LastModifiedTime: unixMillisToTime(int64(t.LastModifiedTime)),
- }
- if t.Schema != nil {
- md.Schema = convertTableSchema(t.Schema)
- }
- if t.View != nil {
- md.View = t.View.Query
- }
- if t.TimePartitioning != nil {
- md.TimePartitioning = &TimePartitioning{
- Expiration: time.Duration(t.TimePartitioning.ExpirationMs) * time.Millisecond,
- }
- }
- if t.StreamingBuffer != nil {
- md.StreamingBuffer = &StreamingBuffer{
- EstimatedBytes: t.StreamingBuffer.EstimatedBytes,
- EstimatedRows: t.StreamingBuffer.EstimatedRows,
- OldestEntryTime: unixMillisToTime(int64(t.StreamingBuffer.OldestEntryTime)),
- }
- }
- return md
-}
-
-func bqDatasetToMetadata(d *bq.Dataset) *DatasetMetadata {
- /// TODO(jba): access
- return &DatasetMetadata{
- CreationTime: unixMillisToTime(d.CreationTime),
- LastModifiedTime: unixMillisToTime(d.LastModifiedTime),
- DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
- Description: d.Description,
- Name: d.FriendlyName,
- ID: d.Id,
- Location: d.Location,
- Labels: d.Labels,
- }
-}
-
-// Convert a number of milliseconds since the Unix epoch to a time.Time.
-// Treat an input of zero specially: convert it to the zero time,
-// rather than the start of the epoch.
-func unixMillisToTime(m int64) time.Time {
- if m == 0 {
- return time.Time{}
- }
- return time.Unix(0, m*1e6)
-}
-
-func convertTableReference(tr *bq.TableReference) *Table {
- return &Table{
- ProjectID: tr.ProjectId,
- DatasetID: tr.DatasetId,
- TableID: tr.TableId,
- }
-}
-
-// patchTableConf contains fields to be patched.
-type patchTableConf struct {
- // These fields are omitted from the patch operation if nil.
- Description *string
- Name *string
- Schema Schema
-}
-
-func (s *bigqueryService) patchTable(ctx context.Context, projectID, datasetID, tableID string, conf *patchTableConf) (*TableMetadata, error) {
- t := &bq.Table{}
- forceSend := func(field string) {
- t.ForceSendFields = append(t.ForceSendFields, field)
- }
-
- if conf.Description != nil {
- t.Description = *conf.Description
- forceSend("Description")
- }
- if conf.Name != nil {
- t.FriendlyName = *conf.Name
- forceSend("FriendlyName")
- }
- if conf.Schema != nil {
- t.Schema = conf.Schema.asTableSchema()
- forceSend("Schema")
- }
- table, err := s.s.Tables.Patch(projectID, datasetID, tableID, t).
- Context(ctx).
- Do()
- if err != nil {
- return nil, err
- }
- return bqTableToMetadata(table), nil
-}
-
-func (s *bigqueryService) insertDataset(ctx context.Context, datasetID, projectID string) error {
- ds := &bq.Dataset{
- DatasetReference: &bq.DatasetReference{DatasetId: datasetID},
- }
- req := s.s.Datasets.Insert(projectID, ds).Context(ctx)
- setClientHeader(req.Header())
- _, err := req.Do()
- return err
-}
-
-func (s *bigqueryService) deleteDataset(ctx context.Context, datasetID, projectID string) error {
- req := s.s.Datasets.Delete(projectID, datasetID).Context(ctx)
- setClientHeader(req.Header())
- return req.Do()
-}
-
-func (s *bigqueryService) getDatasetMetadata(ctx context.Context, projectID, datasetID string) (*DatasetMetadata, error) {
- req := s.s.Datasets.Get(projectID, datasetID).Context(ctx)
- setClientHeader(req.Header())
- table, err := req.Do()
- if err != nil {
- return nil, err
- }
- return bqDatasetToMetadata(table), nil
-}
-
-func (s *bigqueryService) listDatasets(ctx context.Context, projectID string, maxResults int, pageToken string, all bool, filter string) ([]*Dataset, string, error) {
- req := s.s.Datasets.List(projectID).
- Context(ctx).
- PageToken(pageToken).
- All(all)
- setClientHeader(req.Header())
- if maxResults > 0 {
- req.MaxResults(int64(maxResults))
- }
- if filter != "" {
- req.Filter(filter)
- }
- res, err := req.Do()
- if err != nil {
- return nil, "", err
- }
- var datasets []*Dataset
- for _, d := range res.Datasets {
- datasets = append(datasets, s.convertListedDataset(d))
- }
- return datasets, res.NextPageToken, nil
-}
-
-func (s *bigqueryService) convertListedDataset(d *bq.DatasetListDatasets) *Dataset {
- return &Dataset{
- ProjectID: d.DatasetReference.ProjectId,
- DatasetID: d.DatasetReference.DatasetId,
- }
-}
-
-// runWithRetry calls the function until it returns nil or a non-retryable error, or
-// the context is done.
-// See the similar function in ../storage/invoke.go. The main difference is the
-// reason for retrying.
-func runWithRetry(ctx context.Context, call func() error) error {
- backoff := gax.Backoff{
- Initial: 2 * time.Second,
- Max: 32 * time.Second,
- Multiplier: 2,
- }
- return internal.Retry(ctx, backoff, func() (stop bool, err error) {
- err = call()
- if err == nil {
- return true, nil
- }
- return !retryableError(err), err
- })
-}
-
-// Use the criteria in https://cloud.google.com/bigquery/troubleshooting-errors.
-func retryableError(err error) bool {
- e, ok := err.(*googleapi.Error)
- if !ok {
- return false
- }
- var reason string
- if len(e.Errors) > 0 {
- reason = e.Errors[0].Reason
- }
- return reason == "backendError" && (e.Code == 500 || e.Code == 503)
-}
diff --git a/vendor/cloud.google.com/go/bigquery/service_test.go b/vendor/cloud.google.com/go/bigquery/service_test.go
deleted file mode 100644
index 65db4d711..000000000
--- a/vendor/cloud.google.com/go/bigquery/service_test.go
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
- "time"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func TestBQTableToMetadata(t *testing.T) {
- aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
- aTimeMillis := aTime.UnixNano() / 1e6
- for _, test := range []struct {
- in *bq.Table
- want *TableMetadata
- }{
- {&bq.Table{}, &TableMetadata{}}, // test minimal case
- {
- &bq.Table{
- CreationTime: aTimeMillis,
- Description: "desc",
- Etag: "etag",
- ExpirationTime: aTimeMillis,
- FriendlyName: "fname",
- Id: "id",
- LastModifiedTime: uint64(aTimeMillis),
- Location: "loc",
- NumBytes: 123,
- NumLongTermBytes: 23,
- NumRows: 7,
- StreamingBuffer: &bq.Streamingbuffer{
- EstimatedBytes: 11,
- EstimatedRows: 3,
- OldestEntryTime: uint64(aTimeMillis),
- },
- TimePartitioning: &bq.TimePartitioning{
- ExpirationMs: 7890,
- Type: "DAY",
- },
- Type: "EXTERNAL",
- View: &bq.ViewDefinition{Query: "view-query"},
- },
- &TableMetadata{
- Description: "desc",
- Name: "fname",
- View: "view-query",
- ID: "id",
- Type: ExternalTable,
- ExpirationTime: aTime.Truncate(time.Millisecond),
- CreationTime: aTime.Truncate(time.Millisecond),
- LastModifiedTime: aTime.Truncate(time.Millisecond),
- NumBytes: 123,
- NumRows: 7,
- TimePartitioning: &TimePartitioning{Expiration: time.Duration(7890) * time.Millisecond},
- StreamingBuffer: &StreamingBuffer{
- EstimatedBytes: 11,
- EstimatedRows: 3,
- OldestEntryTime: aTime,
- },
- },
- },
- } {
- got := bqTableToMetadata(test.in)
- if !reflect.DeepEqual(got, test.want) {
- t.Errorf("%v:\ngot %+v\nwant %+v", test.in, got, test.want)
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/table.go b/vendor/cloud.google.com/go/bigquery/table.go
deleted file mode 100644
index ffd40bf05..000000000
--- a/vendor/cloud.google.com/go/bigquery/table.go
+++ /dev/null
@@ -1,253 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "fmt"
- "time"
-
- "golang.org/x/net/context"
-
- "cloud.google.com/go/internal/optional"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// A Table is a reference to a BigQuery table.
-type Table struct {
- // ProjectID, DatasetID and TableID may be omitted if the Table is the destination for a query.
- // In this case the result will be stored in an ephemeral table.
- ProjectID string
- DatasetID string
- // TableID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_).
- // The maximum length is 1,024 characters.
- TableID string
-
- c *Client
-}
-
-// TableMetadata contains information about a BigQuery table.
-type TableMetadata struct {
- Description string // The user-friendly description of this table.
- Name string // The user-friendly name for this table.
- Schema Schema
- View string
-
- ID string // An opaque ID uniquely identifying the table.
- Type TableType
-
- // The time when this table expires. If not set, the table will persist
- // indefinitely. Expired tables will be deleted and their storage reclaimed.
- ExpirationTime time.Time
-
- CreationTime time.Time
- LastModifiedTime time.Time
-
- // The size of the table in bytes.
- // This does not include data that is being buffered during a streaming insert.
- NumBytes int64
-
- // The number of rows of data in this table.
- // This does not include data that is being buffered during a streaming insert.
- NumRows uint64
-
- // The time-based partitioning settings for this table.
- TimePartitioning *TimePartitioning
-
- // Contains information regarding this table's streaming buffer, if one is
- // present. This field will be nil if the table is not being streamed to or if
- // there is no data in the streaming buffer.
- StreamingBuffer *StreamingBuffer
-}
-
-// TableCreateDisposition specifies the circumstances under which destination table will be created.
-// Default is CreateIfNeeded.
-type TableCreateDisposition string
-
-const (
- // CreateIfNeeded will create the table if it does not already exist.
- // Tables are created atomically on successful completion of a job.
- CreateIfNeeded TableCreateDisposition = "CREATE_IF_NEEDED"
-
- // CreateNever ensures the table must already exist and will not be
- // automatically created.
- CreateNever TableCreateDisposition = "CREATE_NEVER"
-)
-
-// TableWriteDisposition specifies how existing data in a destination table is treated.
-// Default is WriteAppend.
-type TableWriteDisposition string
-
-const (
- // WriteAppend will append to any existing data in the destination table.
- // Data is appended atomically on successful completion of a job.
- WriteAppend TableWriteDisposition = "WRITE_APPEND"
-
- // WriteTruncate overrides the existing data in the destination table.
- // Data is overwritten atomically on successful completion of a job.
- WriteTruncate TableWriteDisposition = "WRITE_TRUNCATE"
-
- // WriteEmpty fails writes if the destination table already contains data.
- WriteEmpty TableWriteDisposition = "WRITE_EMPTY"
-)
-
-// TableType is the type of table.
-type TableType string
-
-const (
- RegularTable TableType = "TABLE"
- ViewTable TableType = "VIEW"
- ExternalTable TableType = "EXTERNAL"
-)
-
-// StreamingBuffer holds information about the streaming buffer.
-type StreamingBuffer struct {
- // A lower-bound estimate of the number of bytes currently in the streaming
- // buffer.
- EstimatedBytes uint64
-
- // A lower-bound estimate of the number of rows currently in the streaming
- // buffer.
- EstimatedRows uint64
-
- // The time of the oldest entry in the streaming buffer.
- OldestEntryTime time.Time
-}
-
-func (t *Table) tableRefProto() *bq.TableReference {
- return &bq.TableReference{
- ProjectId: t.ProjectID,
- DatasetId: t.DatasetID,
- TableId: t.TableID,
- }
-}
-
-// FullyQualifiedName returns the ID of the table in projectID:datasetID.tableID format.
-func (t *Table) FullyQualifiedName() string {
- return fmt.Sprintf("%s:%s.%s", t.ProjectID, t.DatasetID, t.TableID)
-}
-
-// implicitTable reports whether Table is an empty placeholder, which signifies that a new table should be created with an auto-generated Table ID.
-func (t *Table) implicitTable() bool {
- return t.ProjectID == "" && t.DatasetID == "" && t.TableID == ""
-}
-
-// Create creates a table in the BigQuery service.
-func (t *Table) Create(ctx context.Context, options ...CreateTableOption) error {
- conf := &createTableConf{
- projectID: t.ProjectID,
- datasetID: t.DatasetID,
- tableID: t.TableID,
- }
- for _, o := range options {
- o.customizeCreateTable(conf)
- }
- return t.c.service.createTable(ctx, conf)
-}
-
-// Metadata fetches the metadata for the table.
-func (t *Table) Metadata(ctx context.Context) (*TableMetadata, error) {
- return t.c.service.getTableMetadata(ctx, t.ProjectID, t.DatasetID, t.TableID)
-}
-
-// Delete deletes the table.
-func (t *Table) Delete(ctx context.Context) error {
- return t.c.service.deleteTable(ctx, t.ProjectID, t.DatasetID, t.TableID)
-}
-
-// A CreateTableOption is an optional argument to CreateTable.
-type CreateTableOption interface {
- customizeCreateTable(*createTableConf)
-}
-
-type tableExpiration time.Time
-
-// TableExpiration returns a CreateTableOption that will cause the created table to be deleted after the expiration time.
-func TableExpiration(exp time.Time) CreateTableOption { return tableExpiration(exp) }
-
-func (opt tableExpiration) customizeCreateTable(conf *createTableConf) {
- conf.expiration = time.Time(opt)
-}
-
-type viewQuery string
-
-// ViewQuery returns a CreateTableOption that causes the created table to be a virtual table defined by the supplied query.
-// For more information see: https://cloud.google.com/bigquery/querying-data#views
-func ViewQuery(query string) CreateTableOption { return viewQuery(query) }
-
-func (opt viewQuery) customizeCreateTable(conf *createTableConf) {
- conf.viewQuery = string(opt)
-}
-
-type useStandardSQL struct{}
-
-// UseStandardSQL returns a CreateTableOption to set the table to use standard SQL.
-// The default setting is false (using legacy SQL).
-func UseStandardSQL() CreateTableOption { return useStandardSQL{} }
-
-func (opt useStandardSQL) customizeCreateTable(conf *createTableConf) {
- conf.useStandardSQL = true
-}
-
-// TimePartitioning is a CreateTableOption that can be used to set time-based
-// date partitioning on a table.
-// For more information see: https://cloud.google.com/bigquery/docs/creating-partitioned-tables
-type TimePartitioning struct {
- // (Optional) The amount of time to keep the storage for a partition.
- // If the duration is empty (0), the data in the partitions do not expire.
- Expiration time.Duration
-}
-
-func (opt TimePartitioning) customizeCreateTable(conf *createTableConf) {
- conf.timePartitioning = &opt
-}
-
-// Read fetches the contents of the table.
-func (t *Table) Read(ctx context.Context) *RowIterator {
- return newRowIterator(ctx, t.c.service, &readTableConf{
- projectID: t.ProjectID,
- datasetID: t.DatasetID,
- tableID: t.TableID,
- })
-}
-
-// Update modifies specific Table metadata fields.
-func (t *Table) Update(ctx context.Context, tm TableMetadataToUpdate) (*TableMetadata, error) {
- var conf patchTableConf
- if tm.Description != nil {
- s := optional.ToString(tm.Description)
- conf.Description = &s
- }
- if tm.Name != nil {
- s := optional.ToString(tm.Name)
- conf.Name = &s
- }
- conf.Schema = tm.Schema
- return t.c.service.patchTable(ctx, t.ProjectID, t.DatasetID, t.TableID, &conf)
-}
-
-// TableMetadataToUpdate is used when updating a table's metadata.
-// Only non-nil fields will be updated.
-type TableMetadataToUpdate struct {
- // Description is the user-friendly description of this table.
- Description optional.String
-
- // Name is the user-friendly name for this table.
- Name optional.String
-
- // Schema is the table's schema.
- // When updating a schema, you can add columns but not remove them.
- Schema Schema
- // TODO(jba): support updating the view
-}
diff --git a/vendor/cloud.google.com/go/bigquery/uploader.go b/vendor/cloud.google.com/go/bigquery/uploader.go
deleted file mode 100644
index a0f7ba98e..000000000
--- a/vendor/cloud.google.com/go/bigquery/uploader.go
+++ /dev/null
@@ -1,177 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "errors"
- "fmt"
- "reflect"
-
- "golang.org/x/net/context"
-)
-
-// An Uploader does streaming inserts into a BigQuery table.
-// It is safe for concurrent use.
-type Uploader struct {
- t *Table
-
- // SkipInvalidRows causes rows containing invalid data to be silently
- // ignored. The default value is false, which causes the entire request to
- // fail if there is an attempt to insert an invalid row.
- SkipInvalidRows bool
-
- // IgnoreUnknownValues causes values not matching the schema to be ignored.
- // The default value is false, which causes records containing such values
- // to be treated as invalid records.
- IgnoreUnknownValues bool
-
- // A TableTemplateSuffix allows Uploaders to create tables automatically.
- //
- // Experimental: this option is experimental and may be modified or removed in future versions,
- // regardless of any other documented package stability guarantees.
- //
- // When you specify a suffix, the table you upload data to
- // will be used as a template for creating a new table, with the same schema,
- // called <table> + <suffix>.
- //
- // More information is available at
- // https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
- TableTemplateSuffix string
-}
-
-// Uploader returns an Uploader that can be used to append rows to t.
-// The returned Uploader may optionally be further configured before its Put method is called.
-func (t *Table) Uploader() *Uploader {
- return &Uploader{t: t}
-}
-
-// Put uploads one or more rows to the BigQuery service.
-//
-// If src is ValueSaver, then its Save method is called to produce a row for uploading.
-//
-// If src is a struct or pointer to a struct, then a schema is inferred from it
-// and used to create a StructSaver. The InsertID of the StructSaver will be
-// empty.
-//
-// If src is a slice of ValueSavers, structs, or struct pointers, then each
-// element of the slice is treated as above, and multiple rows are uploaded.
-//
-// Put returns a PutMultiError if one or more rows failed to be uploaded.
-// The PutMultiError contains a RowInsertionError for each failed row.
-//
-// Put will retry on temporary errors (see
-// https://cloud.google.com/bigquery/troubleshooting-errors). This can result
-// in duplicate rows if you do not use insert IDs. Also, if the error persists,
-// the call will run indefinitely. Pass a context with a timeout to prevent
-// hanging calls.
-func (u *Uploader) Put(ctx context.Context, src interface{}) error {
- savers, err := valueSavers(src)
- if err != nil {
- return err
- }
- return u.putMulti(ctx, savers)
-}
-
-func valueSavers(src interface{}) ([]ValueSaver, error) {
- saver, ok, err := toValueSaver(src)
- if err != nil {
- return nil, err
- }
- if ok {
- return []ValueSaver{saver}, nil
- }
- srcVal := reflect.ValueOf(src)
- if srcVal.Kind() != reflect.Slice {
- return nil, fmt.Errorf("%T is not a ValueSaver, struct, struct pointer, or slice", src)
-
- }
- var savers []ValueSaver
- for i := 0; i < srcVal.Len(); i++ {
- s := srcVal.Index(i).Interface()
- saver, ok, err := toValueSaver(s)
- if err != nil {
- return nil, err
- }
- if !ok {
- return nil, fmt.Errorf("src[%d] has type %T, which is not a ValueSaver, struct or struct pointer", i, s)
- }
- savers = append(savers, saver)
- }
- return savers, nil
-}
-
-// Make a ValueSaver from x, which must implement ValueSaver already
-// or be a struct or pointer to struct.
-func toValueSaver(x interface{}) (ValueSaver, bool, error) {
- if _, ok := x.(StructSaver); ok {
- return nil, false, errors.New("bigquery: use &StructSaver, not StructSaver")
- }
- var insertID string
- // Handle StructSavers specially so we can infer the schema if necessary.
- if ss, ok := x.(*StructSaver); ok && ss.Schema == nil {
- x = ss.Struct
- insertID = ss.InsertID
- // Fall through so we can infer the schema.
- }
- if saver, ok := x.(ValueSaver); ok {
- return saver, ok, nil
- }
- v := reflect.ValueOf(x)
- // Support Put with []interface{}
- if v.Kind() == reflect.Interface {
- v = v.Elem()
- }
- if v.Kind() == reflect.Ptr {
- v = v.Elem()
- }
- if v.Kind() != reflect.Struct {
- return nil, false, nil
- }
- schema, err := inferSchemaReflectCached(v.Type())
- if err != nil {
- return nil, false, err
- }
- return &StructSaver{
- Struct: x,
- InsertID: insertID,
- Schema: schema,
- }, true, nil
-}
-
-func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
- var rows []*insertionRow
- for _, saver := range src {
- row, insertID, err := saver.Save()
- if err != nil {
- return err
- }
- rows = append(rows, &insertionRow{InsertID: insertID, Row: row})
- }
-
- return u.t.c.service.insertRows(ctx, u.t.ProjectID, u.t.DatasetID, u.t.TableID, rows, &insertRowsConf{
- skipInvalidRows: u.SkipInvalidRows,
- ignoreUnknownValues: u.IgnoreUnknownValues,
- templateSuffix: u.TableTemplateSuffix,
- })
-}
-
-// An insertionRow represents a row of data to be inserted into a table.
-type insertionRow struct {
- // If InsertID is non-empty, BigQuery will use it to de-duplicate insertions of
- // this row on a best-effort basis.
- InsertID string
- // The data to be inserted, represented as a map from field name to Value.
- Row map[string]Value
-}
diff --git a/vendor/cloud.google.com/go/bigquery/uploader_test.go b/vendor/cloud.google.com/go/bigquery/uploader_test.go
deleted file mode 100644
index 6d6745f98..000000000
--- a/vendor/cloud.google.com/go/bigquery/uploader_test.go
+++ /dev/null
@@ -1,279 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "reflect"
- "testing"
-
- "cloud.google.com/go/internal/pretty"
-
- "golang.org/x/net/context"
-)
-
-type testSaver struct {
- ir *insertionRow
- err error
-}
-
-func (ts testSaver) Save() (map[string]Value, string, error) {
- return ts.ir.Row, ts.ir.InsertID, ts.err
-}
-
-func TestRejectsNonValueSavers(t *testing.T) {
- client := &Client{projectID: "project-id"}
- u := Uploader{t: client.Dataset("dataset-id").Table("table-id")}
- inputs := []interface{}{
- 1,
- []int{1, 2},
- []interface{}{
- testSaver{ir: &insertionRow{"a", map[string]Value{"one": 1}}},
- 1,
- },
- StructSaver{},
- }
- for _, in := range inputs {
- if err := u.Put(context.Background(), in); err == nil {
- t.Errorf("put value: %v; got nil, want error", in)
- }
- }
-}
-
-type insertRowsRecorder struct {
- rowBatches [][]*insertionRow
- service
-}
-
-func (irr *insertRowsRecorder) insertRows(ctx context.Context, projectID, datasetID, tableID string, rows []*insertionRow, conf *insertRowsConf) error {
- irr.rowBatches = append(irr.rowBatches, rows)
- return nil
-}
-
-func TestInsertsData(t *testing.T) {
- testCases := []struct {
- data [][]*insertionRow
- }{
- {
- data: [][]*insertionRow{
- {
- &insertionRow{"a", map[string]Value{"one": 1}},
- },
- },
- },
- {
-
- data: [][]*insertionRow{
- {
- &insertionRow{"a", map[string]Value{"one": 1}},
- &insertionRow{"b", map[string]Value{"two": 2}},
- },
- },
- },
- {
-
- data: [][]*insertionRow{
- {
- &insertionRow{"a", map[string]Value{"one": 1}},
- },
- {
- &insertionRow{"b", map[string]Value{"two": 2}},
- },
- },
- },
- {
-
- data: [][]*insertionRow{
- {
- &insertionRow{"a", map[string]Value{"one": 1}},
- &insertionRow{"b", map[string]Value{"two": 2}},
- },
- {
- &insertionRow{"c", map[string]Value{"three": 3}},
- &insertionRow{"d", map[string]Value{"four": 4}},
- },
- },
- },
- }
- for _, tc := range testCases {
- irr := &insertRowsRecorder{}
- client := &Client{
- projectID: "project-id",
- service: irr,
- }
- u := client.Dataset("dataset-id").Table("table-id").Uploader()
- for _, batch := range tc.data {
- if len(batch) == 0 {
- continue
- }
- var toUpload interface{}
- if len(batch) == 1 {
- toUpload = testSaver{ir: batch[0]}
- } else {
- savers := []testSaver{}
- for _, row := range batch {
- savers = append(savers, testSaver{ir: row})
- }
- toUpload = savers
- }
-
- err := u.Put(context.Background(), toUpload)
- if err != nil {
- t.Errorf("expected successful Put of ValueSaver; got: %v", err)
- }
- }
- if got, want := irr.rowBatches, tc.data; !reflect.DeepEqual(got, want) {
- t.Errorf("got: %v, want: %v", got, want)
- }
- }
-}
-
-type uploadOptionRecorder struct {
- received *insertRowsConf
- service
-}
-
-func (u *uploadOptionRecorder) insertRows(ctx context.Context, projectID, datasetID, tableID string, rows []*insertionRow, conf *insertRowsConf) error {
- u.received = conf
- return nil
-}
-
-func TestUploadOptionsPropagate(t *testing.T) {
- // we don't care for the data in this testcase.
- dummyData := testSaver{ir: &insertionRow{}}
- recorder := new(uploadOptionRecorder)
- c := &Client{service: recorder}
- table := &Table{
- ProjectID: "project-id",
- DatasetID: "dataset-id",
- TableID: "table-id",
- c: c,
- }
-
- tests := [...]struct {
- ul *Uploader
- conf insertRowsConf
- }{
- {
- // test zero options lead to zero value for insertRowsConf
- ul: table.Uploader(),
- },
- {
- ul: func() *Uploader {
- u := table.Uploader()
- u.TableTemplateSuffix = "suffix"
- return u
- }(),
- conf: insertRowsConf{
- templateSuffix: "suffix",
- },
- },
- {
- ul: func() *Uploader {
- u := table.Uploader()
- u.IgnoreUnknownValues = true
- return u
- }(),
- conf: insertRowsConf{
- ignoreUnknownValues: true,
- },
- },
- {
- ul: func() *Uploader {
- u := table.Uploader()
- u.SkipInvalidRows = true
- return u
- }(),
- conf: insertRowsConf{
- skipInvalidRows: true,
- },
- },
- { // multiple upload options combine
- ul: func() *Uploader {
- u := table.Uploader()
- u.TableTemplateSuffix = "suffix"
- u.IgnoreUnknownValues = true
- u.SkipInvalidRows = true
- return u
- }(),
- conf: insertRowsConf{
- templateSuffix: "suffix",
- skipInvalidRows: true,
- ignoreUnknownValues: true,
- },
- },
- }
-
- for i, tc := range tests {
- err := tc.ul.Put(context.Background(), dummyData)
- if err != nil {
- t.Fatalf("%d: expected successful Put of ValueSaver; got: %v", i, err)
- }
-
- if recorder.received == nil {
- t.Fatalf("%d: received no options at all!", i)
- }
-
- want := tc.conf
- got := *recorder.received
- if got != want {
- t.Errorf("%d: got %#v, want %#v, ul=%#v", i, got, want, tc.ul)
- }
- }
-}
-
-func TestValueSavers(t *testing.T) {
- ts := &testSaver{ir: &insertionRow{}}
- type T struct{ I int }
- schema, err := InferSchema(T{})
- if err != nil {
- t.Fatal(err)
- }
- for _, test := range []struct {
- in interface{}
- want []ValueSaver
- }{
- {ts, []ValueSaver{ts}},
- {T{I: 1}, []ValueSaver{&StructSaver{Schema: schema, Struct: T{I: 1}}}},
- {[]ValueSaver{ts, ts}, []ValueSaver{ts, ts}},
- {[]interface{}{ts, ts}, []ValueSaver{ts, ts}},
- {[]T{{I: 1}, {I: 2}}, []ValueSaver{
- &StructSaver{Schema: schema, Struct: T{I: 1}},
- &StructSaver{Schema: schema, Struct: T{I: 2}},
- }},
- {[]interface{}{T{I: 1}, &T{I: 2}}, []ValueSaver{
- &StructSaver{Schema: schema, Struct: T{I: 1}},
- &StructSaver{Schema: schema, Struct: &T{I: 2}},
- }},
- {&StructSaver{Struct: T{I: 3}, InsertID: "foo"},
- []ValueSaver{
- &StructSaver{Schema: schema, Struct: T{I: 3}, InsertID: "foo"},
- }},
- } {
- got, err := valueSavers(test.in)
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(got, test.want) {
- t.Errorf("%+v: got %v, want %v", test.in, pretty.Value(got), pretty.Value(test.want))
- }
- // Make sure Save is successful.
- for i, vs := range got {
- _, _, err := vs.Save()
- if err != nil {
- t.Fatalf("%+v, #%d: got error %v, want nil", test.in, i, err)
- }
- }
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/utils_test.go b/vendor/cloud.google.com/go/bigquery/utils_test.go
deleted file mode 100644
index c781f9c3f..000000000
--- a/vendor/cloud.google.com/go/bigquery/utils_test.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "golang.org/x/net/context"
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func defaultGCS() *GCSReference {
- return &GCSReference{
- uris: []string{"uri"},
- }
-}
-
-var defaultQuery = &QueryConfig{
- Q: "query string",
- DefaultProjectID: "def-project-id",
- DefaultDatasetID: "def-dataset-id",
-}
-
-type testService struct {
- *bq.Job
-
- service
-}
-
-func (s *testService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
- s.Job = conf.job
- return &Job{}, nil
-}
-
-func (s *testService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
- return &JobStatus{State: Done}, nil
-}
diff --git a/vendor/cloud.google.com/go/bigquery/value.go b/vendor/cloud.google.com/go/bigquery/value.go
deleted file mode 100644
index 058c4db69..000000000
--- a/vendor/cloud.google.com/go/bigquery/value.go
+++ /dev/null
@@ -1,657 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "encoding/base64"
- "errors"
- "fmt"
- "reflect"
- "strconv"
- "time"
-
- "cloud.google.com/go/civil"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-// Value stores the contents of a single cell from a BigQuery result.
-type Value interface{}
-
-// ValueLoader stores a slice of Values representing a result row from a Read operation.
-// See RowIterator.Next for more information.
-type ValueLoader interface {
- Load(v []Value, s Schema) error
-}
-
-// valueList converts a []Value to implement ValueLoader.
-type valueList []Value
-
-// Load stores a sequence of values in a valueList.
-// It resets the slice length to zero, then appends each value to it.
-func (vs *valueList) Load(v []Value, _ Schema) error {
- *vs = append((*vs)[:0], v...)
- return nil
-}
-
-// valueMap converts a map[string]Value to implement ValueLoader.
-type valueMap map[string]Value
-
-// Load stores a sequence of values in a valueMap.
-func (vm *valueMap) Load(v []Value, s Schema) error {
- if *vm == nil {
- *vm = map[string]Value{}
- }
- loadMap(*vm, v, s)
- return nil
-}
-
-func loadMap(m map[string]Value, vals []Value, s Schema) {
- for i, f := range s {
- val := vals[i]
- var v interface{}
- switch {
- case f.Schema == nil:
- v = val
- case !f.Repeated:
- m2 := map[string]Value{}
- loadMap(m2, val.([]Value), f.Schema)
- v = m2
- default: // repeated and nested
- sval := val.([]Value)
- vs := make([]Value, len(sval))
- for j, e := range sval {
- m2 := map[string]Value{}
- loadMap(m2, e.([]Value), f.Schema)
- vs[j] = m2
- }
- v = vs
- }
- m[f.Name] = v
- }
-}
-
-type structLoader struct {
- typ reflect.Type // type of struct
- err error
- ops []structLoaderOp
-
- vstructp reflect.Value // pointer to current struct value; changed by set
-}
-
-// A setFunc is a function that sets a struct field or slice/array
-// element to a value.
-type setFunc func(v reflect.Value, val interface{}) error
-
-// A structLoaderOp instructs the loader to set a struct field to a row value.
-type structLoaderOp struct {
- fieldIndex []int
- valueIndex int
- setFunc setFunc
- repeated bool
-}
-
-var errNoNulls = errors.New("bigquery: NULL values cannot be read into structs")
-
-func setAny(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- v.Set(reflect.ValueOf(x))
- return nil
-}
-
-func setInt(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- xx := x.(int64)
- if v.OverflowInt(xx) {
- return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
- }
- v.SetInt(xx)
- return nil
-}
-
-func setFloat(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- xx := x.(float64)
- if v.OverflowFloat(xx) {
- return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
- }
- v.SetFloat(xx)
- return nil
-}
-
-func setBool(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- v.SetBool(x.(bool))
- return nil
-}
-
-func setString(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- v.SetString(x.(string))
- return nil
-}
-
-func setBytes(v reflect.Value, x interface{}) error {
- if x == nil {
- return errNoNulls
- }
- v.SetBytes(x.([]byte))
- return nil
-}
-
-// set remembers a value for the next call to Load. The value must be
-// a pointer to a struct. (This is checked in RowIterator.Next.)
-func (sl *structLoader) set(structp interface{}, schema Schema) error {
- if sl.err != nil {
- return sl.err
- }
- sl.vstructp = reflect.ValueOf(structp)
- typ := sl.vstructp.Type().Elem()
- if sl.typ == nil {
- // First call: remember the type and compile the schema.
- sl.typ = typ
- ops, err := compileToOps(typ, schema)
- if err != nil {
- sl.err = err
- return err
- }
- sl.ops = ops
- } else if sl.typ != typ {
- return fmt.Errorf("bigquery: struct type changed from %s to %s", sl.typ, typ)
- }
- return nil
-}
-
-// compileToOps produces a sequence of operations that will set the fields of a
-// value of structType to the contents of a row with schema.
-func compileToOps(structType reflect.Type, schema Schema) ([]structLoaderOp, error) {
- var ops []structLoaderOp
- fields, err := fieldCache.Fields(structType)
- if err != nil {
- return nil, err
- }
- for i, schemaField := range schema {
- // Look for an exported struct field with the same name as the schema
- // field, ignoring case (BigQuery column names are case-insensitive,
- // and we want to act like encoding/json anyway).
- structField := fields.Match(schemaField.Name)
- if structField == nil {
- // Ignore schema fields with no corresponding struct field.
- continue
- }
- op := structLoaderOp{
- fieldIndex: structField.Index,
- valueIndex: i,
- }
- t := structField.Type
- if schemaField.Repeated {
- if t.Kind() != reflect.Slice && t.Kind() != reflect.Array {
- return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but struct field %s has type %s",
- schemaField.Name, structField.Name, t)
- }
- t = t.Elem()
- op.repeated = true
- }
- if schemaField.Type == RecordFieldType {
- // Field can be a struct or a pointer to a struct.
- if t.Kind() == reflect.Ptr {
- t = t.Elem()
- }
- if t.Kind() != reflect.Struct {
- return nil, fmt.Errorf("bigquery: field %s has type %s, expected struct or *struct",
- structField.Name, structField.Type)
- }
- nested, err := compileToOps(t, schemaField.Schema)
- if err != nil {
- return nil, err
- }
- op.setFunc = func(v reflect.Value, val interface{}) error {
- return setNested(nested, v, val.([]Value))
- }
- } else {
- op.setFunc = determineSetFunc(t, schemaField.Type)
- if op.setFunc == nil {
- return nil, fmt.Errorf("bigquery: schema field %s of type %s is not assignable to struct field %s of type %s",
- schemaField.Name, schemaField.Type, structField.Name, t)
- }
- }
- ops = append(ops, op)
- }
- return ops, nil
-}
-
-// determineSetFunc chooses the best function for setting a field of type ftype
-// to a value whose schema field type is sftype. It returns nil if stype
-// is not assignable to ftype.
-// determineSetFunc considers only basic types. See compileToOps for
-// handling of repetition and nesting.
-func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
- switch stype {
- case StringFieldType:
- if ftype.Kind() == reflect.String {
- return setString
- }
-
- case BytesFieldType:
- if ftype == typeOfByteSlice {
- return setBytes
- }
-
- case IntegerFieldType:
- if isSupportedIntType(ftype) {
- return setInt
- }
-
- case FloatFieldType:
- switch ftype.Kind() {
- case reflect.Float32, reflect.Float64:
- return setFloat
- }
-
- case BooleanFieldType:
- if ftype.Kind() == reflect.Bool {
- return setBool
- }
-
- case TimestampFieldType:
- if ftype == typeOfGoTime {
- return setAny
- }
-
- case DateFieldType:
- if ftype == typeOfDate {
- return setAny
- }
-
- case TimeFieldType:
- if ftype == typeOfTime {
- return setAny
- }
-
- case DateTimeFieldType:
- if ftype == typeOfDateTime {
- return setAny
- }
- }
- return nil
-}
-
-func (sl *structLoader) Load(values []Value, _ Schema) error {
- if sl.err != nil {
- return sl.err
- }
- return runOps(sl.ops, sl.vstructp.Elem(), values)
-}
-
-// runOps executes a sequence of ops, setting the fields of vstruct to the
-// supplied values.
-func runOps(ops []structLoaderOp, vstruct reflect.Value, values []Value) error {
- for _, op := range ops {
- field := vstruct.FieldByIndex(op.fieldIndex)
- var err error
- if op.repeated {
- err = setRepeated(field, values[op.valueIndex].([]Value), op.setFunc)
- } else {
- err = op.setFunc(field, values[op.valueIndex])
- }
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func setNested(ops []structLoaderOp, v reflect.Value, vals []Value) error {
- // v is either a struct or a pointer to a struct.
- if v.Kind() == reflect.Ptr {
- // If the pointer is nil, set it to a zero struct value.
- if v.IsNil() {
- v.Set(reflect.New(v.Type().Elem()))
- }
- v = v.Elem()
- }
- return runOps(ops, v, vals)
-}
-
-func setRepeated(field reflect.Value, vslice []Value, setElem setFunc) error {
- vlen := len(vslice)
- var flen int
- switch field.Type().Kind() {
- case reflect.Slice:
- // Make a slice of the right size, avoiding allocation if possible.
- switch {
- case field.Len() < vlen:
- field.Set(reflect.MakeSlice(field.Type(), vlen, vlen))
- case field.Len() > vlen:
- field.SetLen(vlen)
- }
- flen = vlen
-
- case reflect.Array:
- flen = field.Len()
- if flen > vlen {
- // Set extra elements to their zero value.
- z := reflect.Zero(field.Type().Elem())
- for i := vlen; i < flen; i++ {
- field.Index(i).Set(z)
- }
- }
- default:
- return fmt.Errorf("bigquery: impossible field type %s", field.Type())
- }
- for i, val := range vslice {
- if i < flen { // avoid writing past the end of a short array
- if err := setElem(field.Index(i), val); err != nil {
- return err
- }
- }
- }
- return nil
-}
-
-// A ValueSaver returns a row of data to be inserted into a table.
-type ValueSaver interface {
- // Save returns a row to be inserted into a BigQuery table, represented
- // as a map from field name to Value.
- // If insertID is non-empty, BigQuery will use it to de-duplicate
- // insertions of this row on a best-effort basis.
- Save() (row map[string]Value, insertID string, err error)
-}
-
-// ValuesSaver implements ValueSaver for a slice of Values.
-type ValuesSaver struct {
- Schema Schema
-
- // If non-empty, BigQuery will use InsertID to de-duplicate insertions
- // of this row on a best-effort basis.
- InsertID string
-
- Row []Value
-}
-
-// Save implements ValueSaver.
-func (vls *ValuesSaver) Save() (map[string]Value, string, error) {
- m, err := valuesToMap(vls.Row, vls.Schema)
- return m, vls.InsertID, err
-}
-
-func valuesToMap(vs []Value, schema Schema) (map[string]Value, error) {
- if len(vs) != len(schema) {
- return nil, errors.New("Schema does not match length of row to be inserted")
- }
-
- m := make(map[string]Value)
- for i, fieldSchema := range schema {
- if fieldSchema.Type != RecordFieldType {
- m[fieldSchema.Name] = vs[i]
- continue
- }
- // Nested record, possibly repeated.
- vals, ok := vs[i].([]Value)
- if !ok {
- return nil, errors.New("nested record is not a []Value")
- }
- if !fieldSchema.Repeated {
- value, err := valuesToMap(vals, fieldSchema.Schema)
- if err != nil {
- return nil, err
- }
- m[fieldSchema.Name] = value
- continue
- }
- // A repeated nested field is converted into a slice of maps.
- var maps []Value
- for _, v := range vals {
- sv, ok := v.([]Value)
- if !ok {
- return nil, errors.New("nested record in slice is not a []Value")
- }
- value, err := valuesToMap(sv, fieldSchema.Schema)
- if err != nil {
- return nil, err
- }
- maps = append(maps, value)
- }
- m[fieldSchema.Name] = maps
- }
- return m, nil
-}
-
-// StructSaver implements ValueSaver for a struct.
-// The struct is converted to a map of values by using the values of struct
-// fields corresponding to schema fields. Additional and missing
-// fields are ignored, as are nested struct pointers that are nil.
-type StructSaver struct {
- // Schema determines what fields of the struct are uploaded. It should
- // match the table's schema.
- Schema Schema
-
- // If non-empty, BigQuery will use InsertID to de-duplicate insertions
- // of this row on a best-effort basis.
- InsertID string
-
- // Struct should be a struct or a pointer to a struct.
- Struct interface{}
-}
-
-// Save implements ValueSaver.
-func (ss *StructSaver) Save() (row map[string]Value, insertID string, err error) {
- vstruct := reflect.ValueOf(ss.Struct)
- row, err = structToMap(vstruct, ss.Schema)
- if err != nil {
- return nil, "", err
- }
- return row, ss.InsertID, nil
-}
-
-func structToMap(vstruct reflect.Value, schema Schema) (map[string]Value, error) {
- if vstruct.Kind() == reflect.Ptr {
- vstruct = vstruct.Elem()
- }
- if !vstruct.IsValid() {
- return nil, nil
- }
- m := map[string]Value{}
- if vstruct.Kind() != reflect.Struct {
- return nil, fmt.Errorf("bigquery: type is %s, need struct or struct pointer", vstruct.Type())
- }
- fields, err := fieldCache.Fields(vstruct.Type())
- if err != nil {
- return nil, err
- }
- for _, schemaField := range schema {
- // Look for an exported struct field with the same name as the schema
- // field, ignoring case.
- structField := fields.Match(schemaField.Name)
- if structField == nil {
- continue
- }
- val, err := structFieldToUploadValue(vstruct.FieldByIndex(structField.Index), schemaField)
- if err != nil {
- return nil, err
- }
- // Add the value to the map, unless it is nil.
- if val != nil {
- m[schemaField.Name] = val
- }
- }
- return m, nil
-}
-
-// structFieldToUploadValue converts a struct field to a value suitable for ValueSaver.Save, using
-// the schemaField as a guide.
-// structFieldToUploadValue is careful to return a true nil interface{} when needed, so its
-// caller can easily identify a nil value.
-func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (interface{}, error) {
- if schemaField.Repeated && (vfield.Kind() != reflect.Slice && vfield.Kind() != reflect.Array) {
- return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but value has type %s",
- schemaField.Name, vfield.Type())
- }
-
- // A non-nested field can be represented by its Go value.
- if schemaField.Type != RecordFieldType {
- if !schemaField.Repeated || vfield.Len() > 0 {
- return vfield.Interface(), nil
- }
- // The service treats a null repeated field as an error. Return
- // nil to omit the field entirely.
- return nil, nil
- }
- // A non-repeated nested field is converted into a map[string]Value.
- if !schemaField.Repeated {
- m, err := structToMap(vfield, schemaField.Schema)
- if err != nil {
- return nil, err
- }
- if m == nil {
- return nil, nil
- }
- return m, nil
- }
- // A repeated nested field is converted into a slice of maps.
- if vfield.Len() == 0 {
- return nil, nil
- }
- var vals []Value
- for i := 0; i < vfield.Len(); i++ {
- m, err := structToMap(vfield.Index(i), schemaField.Schema)
- if err != nil {
- return nil, err
- }
- vals = append(vals, m)
- }
- return vals, nil
-}
-
-// convertRows converts a series of TableRows into a series of Value slices.
-// schema is used to interpret the data from rows; its length must match the
-// length of each row.
-func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) {
- var rs [][]Value
- for _, r := range rows {
- row, err := convertRow(r, schema)
- if err != nil {
- return nil, err
- }
- rs = append(rs, row)
- }
- return rs, nil
-}
-
-func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) {
- if len(schema) != len(r.F) {
- return nil, errors.New("schema length does not match row length")
- }
- var values []Value
- for i, cell := range r.F {
- fs := schema[i]
- v, err := convertValue(cell.V, fs.Type, fs.Schema)
- if err != nil {
- return nil, err
- }
- values = append(values, v)
- }
- return values, nil
-}
-
-func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) {
- switch val := val.(type) {
- case nil:
- return nil, nil
- case []interface{}:
- return convertRepeatedRecord(val, typ, schema)
- case map[string]interface{}:
- return convertNestedRecord(val, schema)
- case string:
- return convertBasicType(val, typ)
- default:
- return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ)
- }
-}
-
-func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) {
- var values []Value
- for _, cell := range vals {
- // each cell contains a single entry, keyed by "v"
- val := cell.(map[string]interface{})["v"]
- v, err := convertValue(val, typ, schema)
- if err != nil {
- return nil, err
- }
- values = append(values, v)
- }
- return values, nil
-}
-
-func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) {
- // convertNestedRecord is similar to convertRow, as a record has the same structure as a row.
-
- // Nested records are wrapped in a map with a single key, "f".
- record := val["f"].([]interface{})
- if len(record) != len(schema) {
- return nil, errors.New("schema length does not match record length")
- }
-
- var values []Value
- for i, cell := range record {
- // each cell contains a single entry, keyed by "v"
- val := cell.(map[string]interface{})["v"]
-
- fs := schema[i]
- v, err := convertValue(val, fs.Type, fs.Schema)
- if err != nil {
- return nil, err
- }
- values = append(values, v)
- }
- return values, nil
-}
-
-// convertBasicType returns val as an interface with a concrete type specified by typ.
-func convertBasicType(val string, typ FieldType) (Value, error) {
- switch typ {
- case StringFieldType:
- return val, nil
- case BytesFieldType:
- return base64.StdEncoding.DecodeString(val)
- case IntegerFieldType:
- return strconv.ParseInt(val, 10, 64)
- case FloatFieldType:
- return strconv.ParseFloat(val, 64)
- case BooleanFieldType:
- return strconv.ParseBool(val)
- case TimestampFieldType:
- f, err := strconv.ParseFloat(val, 64)
- return Value(time.Unix(0, int64(f*1e9)).UTC()), err
- case DateFieldType:
- return civil.ParseDate(val)
- case TimeFieldType:
- return civil.ParseTime(val)
- case DateTimeFieldType:
- return civil.ParseDateTime(val)
- default:
- return nil, fmt.Errorf("unrecognized type: %s", typ)
- }
-}
diff --git a/vendor/cloud.google.com/go/bigquery/value_test.go b/vendor/cloud.google.com/go/bigquery/value_test.go
deleted file mode 100644
index 01accd226..000000000
--- a/vendor/cloud.google.com/go/bigquery/value_test.go
+++ /dev/null
@@ -1,885 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package bigquery
-
-import (
- "encoding/base64"
- "fmt"
- "math"
- "reflect"
- "testing"
- "time"
-
- "cloud.google.com/go/civil"
- "cloud.google.com/go/internal/pretty"
-
- bq "google.golang.org/api/bigquery/v2"
-)
-
-func TestConvertBasicValues(t *testing.T) {
- schema := []*FieldSchema{
- {Type: StringFieldType},
- {Type: IntegerFieldType},
- {Type: FloatFieldType},
- {Type: BooleanFieldType},
- {Type: BytesFieldType},
- }
- row := &bq.TableRow{
- F: []*bq.TableCell{
- {V: "a"},
- {V: "1"},
- {V: "1.2"},
- {V: "true"},
- {V: base64.StdEncoding.EncodeToString([]byte("foo"))},
- },
- }
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- want := []Value{"a", int64(1), 1.2, true, []byte("foo")}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting basic values: got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestConvertTime(t *testing.T) {
- // TODO(jba): add tests for civil time types.
- schema := []*FieldSchema{
- {Type: TimestampFieldType},
- }
- thyme := time.Date(1970, 1, 1, 10, 0, 0, 10, time.UTC)
- row := &bq.TableRow{
- F: []*bq.TableCell{
- {V: fmt.Sprintf("%.10f", float64(thyme.UnixNano())/1e9)},
- },
- }
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- if !got[0].(time.Time).Equal(thyme) {
- t.Errorf("converting basic values: got:\n%v\nwant:\n%v", got, thyme)
- }
- if got[0].(time.Time).Location() != time.UTC {
- t.Errorf("expected time zone UTC: got:\n%v", got)
- }
-}
-
-func TestConvertNullValues(t *testing.T) {
- schema := []*FieldSchema{
- {Type: StringFieldType},
- }
- row := &bq.TableRow{
- F: []*bq.TableCell{
- {V: nil},
- },
- }
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- want := []Value{nil}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting null values: got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestBasicRepetition(t *testing.T) {
- schema := []*FieldSchema{
- {Type: IntegerFieldType, Repeated: true},
- }
- row := &bq.TableRow{
- F: []*bq.TableCell{
- {
- V: []interface{}{
- map[string]interface{}{
- "v": "1",
- },
- map[string]interface{}{
- "v": "2",
- },
- map[string]interface{}{
- "v": "3",
- },
- },
- },
- },
- }
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- want := []Value{[]Value{int64(1), int64(2), int64(3)}}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestNestedRecordContainingRepetition(t *testing.T) {
- schema := []*FieldSchema{
- {
- Type: RecordFieldType,
- Schema: Schema{
- {Type: IntegerFieldType, Repeated: true},
- },
- },
- }
- row := &bq.TableRow{
- F: []*bq.TableCell{
- {
- V: map[string]interface{}{
- "f": []interface{}{
- map[string]interface{}{
- "v": []interface{}{
- map[string]interface{}{"v": "1"},
- map[string]interface{}{"v": "2"},
- map[string]interface{}{"v": "3"},
- },
- },
- },
- },
- },
- },
- }
-
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- want := []Value{[]Value{[]Value{int64(1), int64(2), int64(3)}}}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestRepeatedRecordContainingRepetition(t *testing.T) {
- schema := []*FieldSchema{
- {
- Type: RecordFieldType,
- Repeated: true,
- Schema: Schema{
- {Type: IntegerFieldType, Repeated: true},
- },
- },
- }
- row := &bq.TableRow{F: []*bq.TableCell{
- {
- V: []interface{}{ // repeated records.
- map[string]interface{}{ // first record.
- "v": map[string]interface{}{ // pointless single-key-map wrapper.
- "f": []interface{}{ // list of record fields.
- map[string]interface{}{ // only record (repeated ints)
- "v": []interface{}{ // pointless wrapper.
- map[string]interface{}{
- "v": "1",
- },
- map[string]interface{}{
- "v": "2",
- },
- map[string]interface{}{
- "v": "3",
- },
- },
- },
- },
- },
- },
- map[string]interface{}{ // second record.
- "v": map[string]interface{}{
- "f": []interface{}{
- map[string]interface{}{
- "v": []interface{}{
- map[string]interface{}{
- "v": "4",
- },
- map[string]interface{}{
- "v": "5",
- },
- map[string]interface{}{
- "v": "6",
- },
- },
- },
- },
- },
- },
- },
- },
- }}
-
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- want := []Value{ // the row is a list of length 1, containing an entry for the repeated record.
- []Value{ // the repeated record is a list of length 2, containing an entry for each repetition.
- []Value{ // the record is a list of length 1, containing an entry for the repeated integer field.
- []Value{int64(1), int64(2), int64(3)}, // the repeated integer field is a list of length 3.
- },
- []Value{ // second record
- []Value{int64(4), int64(5), int64(6)},
- },
- },
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting repeated records with repeated values: got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestRepeatedRecordContainingRecord(t *testing.T) {
- schema := []*FieldSchema{
- {
- Type: RecordFieldType,
- Repeated: true,
- Schema: Schema{
- {
- Type: StringFieldType,
- },
- {
- Type: RecordFieldType,
- Schema: Schema{
- {Type: IntegerFieldType},
- {Type: StringFieldType},
- },
- },
- },
- },
- }
- row := &bq.TableRow{F: []*bq.TableCell{
- {
- V: []interface{}{ // repeated records.
- map[string]interface{}{ // first record.
- "v": map[string]interface{}{ // pointless single-key-map wrapper.
- "f": []interface{}{ // list of record fields.
- map[string]interface{}{ // first record field (name)
- "v": "first repeated record",
- },
- map[string]interface{}{ // second record field (nested record).
- "v": map[string]interface{}{ // pointless single-key-map wrapper.
- "f": []interface{}{ // nested record fields
- map[string]interface{}{
- "v": "1",
- },
- map[string]interface{}{
- "v": "two",
- },
- },
- },
- },
- },
- },
- },
- map[string]interface{}{ // second record.
- "v": map[string]interface{}{
- "f": []interface{}{
- map[string]interface{}{
- "v": "second repeated record",
- },
- map[string]interface{}{
- "v": map[string]interface{}{
- "f": []interface{}{
- map[string]interface{}{
- "v": "3",
- },
- map[string]interface{}{
- "v": "four",
- },
- },
- },
- },
- },
- },
- },
- },
- },
- }}
-
- got, err := convertRow(row, schema)
- if err != nil {
- t.Fatalf("error converting: %v", err)
- }
- // TODO: test with flattenresults.
- want := []Value{ // the row is a list of length 1, containing an entry for the repeated record.
- []Value{ // the repeated record is a list of length 2, containing an entry for each repetition.
- []Value{ // record contains a string followed by a nested record.
- "first repeated record",
- []Value{
- int64(1),
- "two",
- },
- },
- []Value{ // second record.
- "second repeated record",
- []Value{
- int64(3),
- "four",
- },
- },
- },
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("converting repeated records containing record : got:\n%v\nwant:\n%v", got, want)
- }
-}
-
-func TestValuesSaverConvertsToMap(t *testing.T) {
- testCases := []struct {
- vs ValuesSaver
- want *insertionRow
- }{
- {
- vs: ValuesSaver{
- Schema: []*FieldSchema{
- {Name: "intField", Type: IntegerFieldType},
- {Name: "strField", Type: StringFieldType},
- },
- InsertID: "iid",
- Row: []Value{1, "a"},
- },
- want: &insertionRow{
- InsertID: "iid",
- Row: map[string]Value{"intField": 1, "strField": "a"},
- },
- },
- {
- vs: ValuesSaver{
- Schema: []*FieldSchema{
- {Name: "intField", Type: IntegerFieldType},
- {
- Name: "recordField",
- Type: RecordFieldType,
- Schema: []*FieldSchema{
- {Name: "nestedInt", Type: IntegerFieldType, Repeated: true},
- },
- },
- },
- InsertID: "iid",
- Row: []Value{1, []Value{[]Value{2, 3}}},
- },
- want: &insertionRow{
- InsertID: "iid",
- Row: map[string]Value{
- "intField": 1,
- "recordField": map[string]Value{
- "nestedInt": []Value{2, 3},
- },
- },
- },
- },
- { // repeated nested field
- vs: ValuesSaver{
- Schema: Schema{
- {
- Name: "records",
- Type: RecordFieldType,
- Schema: Schema{
- {Name: "x", Type: IntegerFieldType},
- {Name: "y", Type: IntegerFieldType},
- },
- Repeated: true,
- },
- },
- InsertID: "iid",
- Row: []Value{ // a row is a []Value
- []Value{ // repeated field's value is a []Value
- []Value{1, 2}, // first record of the repeated field
- []Value{3, 4}, // second record
- },
- },
- },
- want: &insertionRow{
- InsertID: "iid",
- Row: map[string]Value{
- "records": []Value{
- map[string]Value{"x": 1, "y": 2},
- map[string]Value{"x": 3, "y": 4},
- },
- },
- },
- },
- }
- for _, tc := range testCases {
- data, insertID, err := tc.vs.Save()
- if err != nil {
- t.Errorf("Expected successful save; got: %v", err)
- }
- got := &insertionRow{insertID, data}
- if !reflect.DeepEqual(got, tc.want) {
- t.Errorf("saving ValuesSaver:\ngot:\n%+v\nwant:\n%+v", got, tc.want)
- }
- }
-}
-
-func TestStructSaver(t *testing.T) {
- schema := Schema{
- {Name: "s", Type: StringFieldType},
- {Name: "r", Type: IntegerFieldType, Repeated: true},
- {Name: "nested", Type: RecordFieldType, Schema: Schema{
- {Name: "b", Type: BooleanFieldType},
- }},
- {Name: "rnested", Type: RecordFieldType, Repeated: true, Schema: Schema{
- {Name: "b", Type: BooleanFieldType},
- }},
- }
-
- type (
- N struct{ B bool }
- T struct {
- S string
- R []int
- Nested *N
- Rnested []*N
- }
- )
-
- check := func(msg string, in interface{}, want map[string]Value) {
- ss := StructSaver{
- Schema: schema,
- InsertID: "iid",
- Struct: in,
- }
- got, gotIID, err := ss.Save()
- if err != nil {
- t.Fatalf("%s: %v", msg, err)
- }
- if wantIID := "iid"; gotIID != wantIID {
- t.Errorf("%s: InsertID: got %q, want %q", msg, gotIID, wantIID)
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("%s:\ngot\n%#v\nwant\n%#v", msg, got, want)
- }
- }
-
- in := T{
- S: "x",
- R: []int{1, 2},
- Nested: &N{B: true},
- Rnested: []*N{{true}, {false}},
- }
- want := map[string]Value{
- "s": "x",
- "r": []int{1, 2},
- "nested": map[string]Value{"b": true},
- "rnested": []Value{map[string]Value{"b": true}, map[string]Value{"b": false}},
- }
- check("all values", in, want)
- check("all values, ptr", &in, want)
- check("empty struct", T{}, map[string]Value{"s": ""})
-
- // Missing and extra fields ignored.
- type T2 struct {
- S string
- // missing R, Nested, RNested
- Extra int
- }
- check("missing and extra", T2{S: "x"}, map[string]Value{"s": "x"})
-
- check("nils in slice", T{Rnested: []*N{{true}, nil, {false}}},
- map[string]Value{
- "s": "",
- "rnested": []Value{map[string]Value{"b": true}, map[string]Value(nil), map[string]Value{"b": false}},
- })
-}
-
-func TestConvertRows(t *testing.T) {
- schema := []*FieldSchema{
- {Type: StringFieldType},
- {Type: IntegerFieldType},
- {Type: FloatFieldType},
- {Type: BooleanFieldType},
- }
- rows := []*bq.TableRow{
- {F: []*bq.TableCell{
- {V: "a"},
- {V: "1"},
- {V: "1.2"},
- {V: "true"},
- }},
- {F: []*bq.TableCell{
- {V: "b"},
- {V: "2"},
- {V: "2.2"},
- {V: "false"},
- }},
- }
- want := [][]Value{
- {"a", int64(1), 1.2, true},
- {"b", int64(2), 2.2, false},
- }
- got, err := convertRows(rows, schema)
- if err != nil {
- t.Fatalf("got %v, want nil", err)
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("\ngot %v\nwant %v", got, want)
- }
-}
-
-func TestValueList(t *testing.T) {
- schema := Schema{
- {Name: "s", Type: StringFieldType},
- {Name: "i", Type: IntegerFieldType},
- {Name: "f", Type: FloatFieldType},
- {Name: "b", Type: BooleanFieldType},
- }
- want := []Value{"x", 7, 3.14, true}
- var got []Value
- vl := (*valueList)(&got)
- if err := vl.Load(want, schema); err != nil {
- t.Fatal(err)
- }
-
- if !reflect.DeepEqual(got, want) {
- t.Errorf("got %+v, want %+v", got, want)
- }
-
- // Load truncates, not appends.
- // https://github.com/GoogleCloudPlatform/google-cloud-go/issues/437
- if err := vl.Load(want, schema); err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(got, want) {
- t.Errorf("got %+v, want %+v", got, want)
- }
-}
-
-func TestValueMap(t *testing.T) {
- ns := Schema{
- {Name: "x", Type: IntegerFieldType},
- {Name: "y", Type: IntegerFieldType},
- }
- schema := Schema{
- {Name: "s", Type: StringFieldType},
- {Name: "i", Type: IntegerFieldType},
- {Name: "f", Type: FloatFieldType},
- {Name: "b", Type: BooleanFieldType},
- {Name: "n", Type: RecordFieldType, Schema: ns},
- {Name: "rn", Type: RecordFieldType, Schema: ns, Repeated: true},
- }
- in := []Value{"x", 7, 3.14, true,
- []Value{1, 2},
- []Value{[]Value{3, 4}, []Value{5, 6}},
- }
- var vm valueMap
- if err := vm.Load(in, schema); err != nil {
- t.Fatal(err)
- }
- want := map[string]Value{
- "s": "x",
- "i": 7,
- "f": 3.14,
- "b": true,
- "n": map[string]Value{"x": 1, "y": 2},
- "rn": []Value{
- map[string]Value{"x": 3, "y": 4},
- map[string]Value{"x": 5, "y": 6},
- },
- }
- if !reflect.DeepEqual(vm, valueMap(want)) {
- t.Errorf("got\n%+v\nwant\n%+v", vm, want)
- }
-
-}
-
-var (
- // For testing StructLoader
- schema2 = Schema{
- {Name: "s", Type: StringFieldType},
- {Name: "s2", Type: StringFieldType},
- {Name: "by", Type: BytesFieldType},
- {Name: "I", Type: IntegerFieldType},
- {Name: "F", Type: FloatFieldType},
- {Name: "B", Type: BooleanFieldType},
- {Name: "TS", Type: TimestampFieldType},
- {Name: "D", Type: DateFieldType},
- {Name: "T", Type: TimeFieldType},
- {Name: "DT", Type: DateTimeFieldType},
- {Name: "nested", Type: RecordFieldType, Schema: Schema{
- {Name: "nestS", Type: StringFieldType},
- {Name: "nestI", Type: IntegerFieldType},
- }},
- {Name: "t", Type: StringFieldType},
- }
-
- testTimestamp = time.Date(2016, 11, 5, 7, 50, 22, 8, time.UTC)
- testDate = civil.Date{2016, 11, 5}
- testTime = civil.Time{7, 50, 22, 8}
- testDateTime = civil.DateTime{testDate, testTime}
-
- testValues = []Value{"x", "y", []byte{1, 2, 3}, int64(7), 3.14, true,
- testTimestamp, testDate, testTime, testDateTime,
- []Value{"nested", int64(17)}, "z"}
-)
-
-type testStruct1 struct {
- B bool
- I int
- times
- S string
- S2 String
- By []byte
- s string
- F float64
- Nested nested
- Tagged string `bigquery:"t"`
-}
-
-type String string
-
-type nested struct {
- NestS string
- NestI int
-}
-
-type times struct {
- TS time.Time
- T civil.Time
- D civil.Date
- DT civil.DateTime
-}
-
-func TestStructLoader(t *testing.T) {
- var ts1 testStruct1
- if err := load(&ts1, schema2, testValues); err != nil {
- t.Fatal(err)
- }
- // Note: the schema field named "s" gets matched to the exported struct
- // field "S", not the unexported "s".
- want := &testStruct1{
- B: true,
- I: 7,
- F: 3.14,
- times: times{TS: testTimestamp, T: testTime, D: testDate, DT: testDateTime},
- S: "x",
- S2: "y",
- By: []byte{1, 2, 3},
- Nested: nested{NestS: "nested", NestI: 17},
- Tagged: "z",
- }
- if !reflect.DeepEqual(&ts1, want) {
- t.Errorf("got %+v, want %+v", pretty.Value(ts1), pretty.Value(*want))
- d, _, err := pretty.Diff(*want, ts1)
- if err == nil {
- t.Logf("diff:\n%s", d)
- }
- }
-
- // Test pointers to nested structs.
- type nestedPtr struct{ Nested *nested }
- var np nestedPtr
- if err := load(&np, schema2, testValues); err != nil {
- t.Fatal(err)
- }
- want2 := &nestedPtr{Nested: &nested{NestS: "nested", NestI: 17}}
- if !reflect.DeepEqual(&np, want2) {
- t.Errorf("got %+v, want %+v", pretty.Value(np), pretty.Value(*want2))
- }
-
- // Existing values should be reused.
- nst := &nested{NestS: "x", NestI: -10}
- np = nestedPtr{Nested: nst}
- if err := load(&np, schema2, testValues); err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(&np, want2) {
- t.Errorf("got %+v, want %+v", pretty.Value(np), pretty.Value(*want2))
- }
- if np.Nested != nst {
- t.Error("nested struct pointers not equal")
- }
-}
-
-type repStruct struct {
- Nums []int
- ShortNums [2]int // to test truncation
- LongNums [5]int // to test padding with zeroes
- Nested []*nested
-}
-
-var (
- repSchema = Schema{
- {Name: "nums", Type: IntegerFieldType, Repeated: true},
- {Name: "shortNums", Type: IntegerFieldType, Repeated: true},
- {Name: "longNums", Type: IntegerFieldType, Repeated: true},
- {Name: "nested", Type: RecordFieldType, Repeated: true, Schema: Schema{
- {Name: "nestS", Type: StringFieldType},
- {Name: "nestI", Type: IntegerFieldType},
- }},
- }
- v123 = []Value{int64(1), int64(2), int64(3)}
- repValues = []Value{v123, v123, v123,
- []Value{
- []Value{"x", int64(1)},
- []Value{"y", int64(2)},
- },
- }
-)
-
-func TestStructLoaderRepeated(t *testing.T) {
- var r1 repStruct
- if err := load(&r1, repSchema, repValues); err != nil {
- t.Fatal(err)
- }
- want := repStruct{
- Nums: []int{1, 2, 3},
- ShortNums: [...]int{1, 2}, // extra values discarded
- LongNums: [...]int{1, 2, 3, 0, 0},
- Nested: []*nested{{"x", 1}, {"y", 2}},
- }
- if !reflect.DeepEqual(r1, want) {
- t.Errorf("got %+v, want %+v", pretty.Value(r1), pretty.Value(want))
- }
-
- r2 := repStruct{
- Nums: []int{-1, -2, -3, -4, -5}, // truncated to zero and appended to
- LongNums: [...]int{-1, -2, -3, -4, -5}, // unset elements are zeroed
- }
- if err := load(&r2, repSchema, repValues); err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(r2, want) {
- t.Errorf("got %+v, want %+v", pretty.Value(r2), pretty.Value(want))
- }
- if got, want := cap(r2.Nums), 5; got != want {
- t.Errorf("cap(r2.Nums) = %d, want %d", got, want)
- }
-
- // Short slice case.
- r3 := repStruct{Nums: []int{-1}}
- if err := load(&r3, repSchema, repValues); err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(r3, want) {
- t.Errorf("got %+v, want %+v", pretty.Value(r3), pretty.Value(want))
- }
- if got, want := cap(r3.Nums), 3; got != want {
- t.Errorf("cap(r3.Nums) = %d, want %d", got, want)
- }
-
-}
-
-func TestStructLoaderOverflow(t *testing.T) {
- type S struct {
- I int16
- F float32
- }
- schema := Schema{
- {Name: "I", Type: IntegerFieldType},
- {Name: "F", Type: FloatFieldType},
- }
- var s S
- if err := load(&s, schema, []Value{int64(math.MaxInt16 + 1), 0}); err == nil {
- t.Error("int: got nil, want error")
- }
- if err := load(&s, schema, []Value{int64(0), math.MaxFloat32 * 2}); err == nil {
- t.Error("float: got nil, want error")
- }
-}
-
-func TestStructLoaderFieldOverlap(t *testing.T) {
- // It's OK if the struct has fields that the schema does not, and vice versa.
- type S1 struct {
- I int
- X [][]int // not in the schema; does not even correspond to a valid BigQuery type
- // many schema fields missing
- }
- var s1 S1
- if err := load(&s1, schema2, testValues); err != nil {
- t.Fatal(err)
- }
- want1 := S1{I: 7}
- if !reflect.DeepEqual(s1, want1) {
- t.Errorf("got %+v, want %+v", pretty.Value(s1), pretty.Value(want1))
- }
-
- // It's even valid to have no overlapping fields at all.
- type S2 struct{ Z int }
-
- var s2 S2
- if err := load(&s2, schema2, testValues); err != nil {
- t.Fatal(err)
- }
- want2 := S2{}
- if !reflect.DeepEqual(s2, want2) {
- t.Errorf("got %+v, want %+v", pretty.Value(s2), pretty.Value(want2))
- }
-}
-
-func TestStructLoaderErrors(t *testing.T) {
- check := func(sp interface{}) {
- var sl structLoader
- err := sl.set(sp, schema2)
- if err == nil {
- t.Errorf("%T: got nil, want error", sp)
- }
- }
-
- type bad1 struct{ F int32 } // wrong type for FLOAT column
- check(&bad1{})
-
- type bad2 struct{ I uint } // unsupported integer type
- check(&bad2{})
-
- // Using more than one struct type with the same structLoader.
- type different struct {
- B bool
- I int
- times
- S string
- s string
- Nums []int
- }
-
- var sl structLoader
- if err := sl.set(&testStruct1{}, schema2); err != nil {
- t.Fatal(err)
- }
- err := sl.set(&different{}, schema2)
- if err == nil {
- t.Error("different struct types: got nil, want error")
- }
-}
-
-func load(pval interface{}, schema Schema, vals []Value) error {
- var sl structLoader
- if err := sl.set(pval, schema); err != nil {
- return err
- }
- return sl.Load(vals, nil)
-}
-
-func BenchmarkStructLoader_NoCompile(b *testing.B) {
- benchmarkStructLoader(b, false)
-}
-
-func BenchmarkStructLoader_Compile(b *testing.B) {
- benchmarkStructLoader(b, true)
-}
-
-func benchmarkStructLoader(b *testing.B, compile bool) {
- var ts1 testStruct1
- for i := 0; i < b.N; i++ {
- var sl structLoader
- for j := 0; j < 10; j++ {
- if err := load(&ts1, schema2, testValues); err != nil {
- b.Fatal(err)
- }
- if !compile {
- sl.typ = nil
- }
- }
- }
-}