diff --git a/bigquery/pom.xml b/bigquery/pom.xml new file mode 100644 index 00000000000..2ba21ae348b --- /dev/null +++ b/bigquery/pom.xml @@ -0,0 +1,81 @@ + + 4.0.0 + com.google.cloud.bigquery.samples + bigquery-samples + jar + + + doc-samples + com.google.cloud + 1.0.0 + .. + + + + + + googleapis + https://google-api-client-libraries.appspot.com/mavenrepo + + + + + + com.google.apis + google-api-services-bigquery + v2-rev158-1.19.0 + + + com.google.oauth-client + google-oauth-client + ${project.oauth.version} + + + com.google.http-client + google-http-client-jackson2 + ${project.http.version} + + + com.google.oauth-client + google-oauth-client-jetty + ${project.oauth.version} + + + junit + junit + + + com.google.code.gson + gson + 2.3.1 + + + + + 1.19.0 + 1.19.0 + UTF-8 + + + + src/main/java + + + src/main/resources + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 5 + 5 + + + + + + diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java new file mode 100644 index 00000000000..774f7834ba4 --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.bigquery.samples; + + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobConfiguration; +import com.google.api.services.bigquery.model.JobConfigurationQuery; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Scanner; + + +/** + * Example of authorizing with BigQuery and reading from a public dataset. + */ +public class AsyncQuerySample extends BigqueryUtils{ + + + // [START main] + /** + * @param args + * @throws IOException + * @throws InterruptedException + */ + public static void main(String[] args) + throws IOException, InterruptedException { + + Scanner scanner = new Scanner(System.in); + System.out.println("Enter your project id: "); + String projectId = scanner.nextLine(); + System.out.println("Enter your query string: "); + String queryString = scanner.nextLine(); + System.out.println("Run query in batch mode? [true|false] "); + boolean batch = Boolean.valueOf(scanner.nextLine()); + System.out.println("Enter how often to check if your job is complete (milliseconds): "); + long waitTime = scanner.nextLong(); + scanner.close(); + Iterator pages = run(projectId, queryString, batch, waitTime); + while(pages.hasNext()){ + printRows(pages.next().getRows(), System.out); + } + + } + // [END main] + + // [START run] + public static Iterator run(String projectId, + String queryString, + boolean batch, + long waitTime) + throws IOException, InterruptedException{ + + Bigquery bigquery = BigqueryServiceFactory.getService(); + + Job query = asyncQuery(bigquery, projectId, queryString, batch); + Bigquery.Jobs.Get getRequest = bigquery.jobs().get( + projectId, query.getJobReference().getJobId()); + + //Poll every waitTime milliseconds, + //retrying at most retries times if there are errors + pollJob(getRequest, waitTime); + + GetQueryResults resultsRequest = bigquery.jobs().getQueryResults( + projectId, query.getJobReference().getJobId()); + + return getPages(resultsRequest); + } + // [END run] + + // [START asyncQuery] + /** + * Inserts an asynchronous query Job for a particular query + * + * @param bigquery an authorized BigQuery client + * @param projectId a String containing the project ID + * @param querySql the actual query string + * @return a reference to the inserted query job + * @throws IOException + */ + public static Job asyncQuery(Bigquery bigquery, + String projectId, + String querySql, + boolean batch) throws IOException { + + JobConfigurationQuery query_config = new JobConfigurationQuery() + .setQuery(querySql); + + if(batch){ + query_config.setPriority("BATCH"); + } + + Job job = new Job().setConfiguration( + new JobConfiguration().setQuery(query_config)); + + return bigquery.jobs().insert(projectId, job).execute(); + } + // [END asyncQuery] + +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java new file mode 100644 index 00000000000..c69212f84d9 --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryServiceFactory.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.bigquery.samples; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.BigqueryScopes; + +import java.io.IOException; +import java.util.Collection; + +public class BigqueryServiceFactory { + + private static Bigquery service = null; + private static Object service_lock = new Object(); + + public static Bigquery getService() throws IOException{ + if(service==null){ + synchronized(service_lock){ + if(service==null){ + service=createAuthorizedClient(); + } + } + } + return service; + } + + // [START get_service] + private static Bigquery createAuthorizedClient() throws IOException { + Collection BIGQUERY_SCOPES = BigqueryScopes.all(); + HttpTransport TRANSPORT = new NetHttpTransport(); + JsonFactory JSON_FACTORY = new JacksonFactory(); + GoogleCredential credential = GoogleCredential.getApplicationDefault(TRANSPORT, JSON_FACTORY); + if(credential.createScopedRequired()){ + credential = credential.createScoped(BIGQUERY_SCOPES); + } + return new Bigquery.Builder(TRANSPORT, JSON_FACTORY, credential).setApplicationName("BigQuery Samples").build(); + } + // [END get_service] + +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java new file mode 100644 index 00000000000..a5e926d4fe3 --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigqueryUtils.java @@ -0,0 +1,147 @@ +/* + Copyright 2015, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.google.cloud.bigquery.samples; + +import com.google.api.client.json.GenericJson; +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.Bigquery.Datasets; +import com.google.api.services.bigquery.BigqueryRequest; +import com.google.api.services.bigquery.model.DatasetList; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.TableCell; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; +import com.google.gson.Gson; + +import java.io.IOException; +import java.io.PrintStream; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class BigqueryUtils { + + // [START print_rows] + public static void printRows(List rows, PrintStream out){ + for (TableRow row : rows) { + for (TableCell field : row.getF()) { + out.printf("%-50s", field.getV()); + } + out.println(); + } + } + // [END print_rows] + + // [START poll_job] + public static Job pollJob(Bigquery.Jobs.Get request, long interval) + throws IOException, InterruptedException{ + Job job = request.execute(); + while(!job.getStatus().getState().equals("DONE")) { + System.out.println("Job is " + + job.getStatus().getState() + + " waiting " + interval + " milliseconds..."); + Thread.sleep(interval); + job = request.execute(); + } + return job; + } + // [END poll_job] + + // [START paging] + public static Iterator getPages(BigqueryRequest request_template){ + + class PageIterator implements Iterator{ + + BigqueryRequest request; + boolean hasNext = true; + + public PageIterator(BigqueryRequest request_template){ + this.request = request_template; + } + + public boolean hasNext() { + return hasNext ; + } + + public T next() { + if(!hasNext){ + throw new NoSuchElementException(); + } + try { + T response = request.execute(); + if (response.containsKey("pageToken")) { + request = request.set("pageToken", response.get("pageToken")); + } else { + hasNext = false; + } + return response; + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + public void remove() { + this.next(); + } + } + + return new PageIterator(request_template); + } + // [END paging] + + // [START load_schema] + public static TableSchema loadSchema(Reader schemaSource){ + TableSchema sourceSchema = new TableSchema(); + + List fields = (new Gson()).>fromJson( + schemaSource, + (new ArrayList()).getClass()); + + sourceSchema.setFields(fields); + + return sourceSchema; + } + // [END load_schema] + + // [START list_datasets] + /** + * Display all BigQuery datasets associated with a project + * + * @param bigquery an authorized BigQuery client + * @param projectId a string containing the current project ID + * @throws IOException + */ + public static void listDatasets(Bigquery bigquery, String projectId) + throws IOException { + Datasets.List datasetRequest = bigquery.datasets().list(projectId); + DatasetList datasetList = datasetRequest.execute(); + if (datasetList.getDatasets() != null) { + List datasets = datasetList.getDatasets(); + System.out.println("Available datasets\n----------------"); + System.out.println(datasets.toString()); + for (DatasetList.Datasets dataset : datasets) { + System.out.format("%s\n", dataset.getDatasetReference().getDatasetId()); + } + } + } + // [END list_datasets] + +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java new file mode 100644 index 00000000000..9c21b2d6488 --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java @@ -0,0 +1,102 @@ +/* + Copyright 2015, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.google.cloud.bigquery.samples; + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobConfiguration; +import com.google.api.services.bigquery.model.JobConfigurationExtract; +import com.google.api.services.bigquery.model.TableReference; + +import java.io.IOException; +import java.util.Scanner; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class ExportDataCloudStorageSample extends BigqueryUtils { + + + + // [START main] + public static void main(String[] args) throws IOException, InterruptedException{ + Scanner scanner = new Scanner(System.in); + System.out.println("Enter your project id: "); + String projectId = scanner.nextLine(); + System.out.println("Enter your dataset id: "); + String datasetId = scanner.nextLine(); + System.out.println("Enter your table id: "); + String tableId = scanner.nextLine(); + System.out.println("Enter the Google Cloud Storage Path to which you'd like to export: "); + String cloudStoragePath = scanner.nextLine(); + System.out.println("Enter how often to check if your job is complete (milliseconds): "); + long interval = scanner.nextLong(); + scanner.close(); + + run(cloudStoragePath, projectId, datasetId, tableId, interval); + + } + // [END main] + + // [START run] + public static void run( + String cloudStoragePath, + String projectId, + String datasetId, + String tableId, + long interval) throws IOException, InterruptedException{ + + Bigquery bigquery = BigqueryServiceFactory.getService(); + + Job extractJob = extractJob( + bigquery, + cloudStoragePath, + new TableReference() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId)); + + Bigquery.Jobs.Get get_job = bigquery.jobs().get( + extractJob.getJobReference().getProjectId(), + extractJob.getJobReference().getJobId()); + + pollJob(get_job, interval); + + System.out.println("Export is Done!"); + + } + // [END run] + + + // [START extract_job] + public static Job extractJob( + Bigquery bigquery, + String cloudStoragePath, + TableReference table) throws IOException{ + + JobConfigurationExtract extract = new JobConfigurationExtract() + .setSourceTable(table) + .setDestinationUri(cloudStoragePath); + + return bigquery.jobs().insert(table.getProjectId(), + new Job().setConfiguration(new JobConfiguration().setExtract(extract))) + .execute(); + } + // [END extract_job] + + + + +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCSVSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCSVSample.java new file mode 100644 index 00000000000..c328bf570ed --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCSVSample.java @@ -0,0 +1,117 @@ +/* + Copyright 2015, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.google.cloud.bigquery.samples; + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobConfiguration; +import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.api.services.bigquery.model.TableReference; +import com.google.api.services.bigquery.model.TableSchema; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.Scanner; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class LoadDataCSVSample extends BigqueryUtils { + + + // [START main] + public static void main(String[] args) throws IOException, InterruptedException{ + Scanner scanner = new Scanner(System.in); + System.out.println("Enter your project id: "); + String projectId = scanner.nextLine(); + System.out.println("Enter your dataset id: "); + String datasetId = scanner.nextLine(); + System.out.println("Enter your table id: "); + String tableId = scanner.nextLine(); + System.out.println("Enter the Google Cloud Storage Path to the data you'd like to load: "); + String cloudStoragePath = scanner.nextLine(); + System.out.println("Enter the filepath to your schema: "); + String sourceSchemaPath = scanner.nextLine(); + + + System.out.println("Enter how often to check if your job is complete (milliseconds): "); + long interval = scanner.nextLong(); + scanner.close(); + + run(cloudStoragePath, + projectId, + datasetId, + tableId, + new FileReader(new File(sourceSchemaPath)), + interval); + + } + // [END main] + + // [START run] + public static void run( + String cloudStoragePath, + String projectId, + String datasetId, + String tableId, + Reader schemaSource, + long interval) throws IOException, InterruptedException{ + + Bigquery bigquery = BigqueryServiceFactory.getService(); + + + Job loadJob = loadJob( + bigquery, + cloudStoragePath, + new TableReference() + .setProjectId(projectId) + .setDatasetId(datasetId) + .setTableId(tableId), + loadSchema(schemaSource)); + + Bigquery.Jobs.Get get_job = bigquery.jobs().get( + loadJob.getJobReference().getProjectId(), + loadJob.getJobReference().getJobId()); + + pollJob(get_job, interval); + + System.out.println("Load is Done!"); + + } + // [END run] + + // [START load_job] + public static Job loadJob( + Bigquery bigquery, + String cloudStoragePath, + TableReference table, + TableSchema schema) throws IOException{ + + JobConfigurationLoad load = new JobConfigurationLoad() + .setDestinationTable(table) + .setSchema(schema) + .setSourceUris(Collections.singletonList(cloudStoragePath)); + + return bigquery.jobs().insert(table.getProjectId(), + new Job().setConfiguration(new JobConfiguration().setLoad(load))) + .execute(); + } + // [END load_job] + + +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java new file mode 100644 index 00000000000..d914f5fa5bc --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java @@ -0,0 +1,136 @@ +/* + Copyright 2015, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.google.cloud.bigquery.samples; + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.TableDataInsertAllRequest; +import com.google.api.services.bigquery.model.TableDataInsertAllResponse; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.gson.stream.JsonReader; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Scanner; + + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class StreamingSample extends BigqueryUtils { + + + + // [START main] + public static void main(String[] args) throws IOException{ + final Scanner scanner = new Scanner(System.in); + System.out.println("Enter your project id: "); + String projectId = scanner.nextLine(); + System.out.println("Enter your dataset id: "); + String datasetId = scanner.nextLine(); + System.out.println("Enter your table id: "); + String tableId = scanner.nextLine(); + scanner.close(); + + System.out.println("Enter JSON to stream to BigQuery: \n" + + "Press End-of-stream (CTRL-D) to stop"); + + JsonReader fromCLI = new JsonReader(new InputStreamReader(System.in)); + + Iterator responses = run(projectId, + datasetId, + tableId, + fromCLI); + + while(responses.hasNext()){ + System.out.println(responses.next()); + } + + fromCLI.close(); + } + // [END main] + + + +// [START run] + public static Iterator run(final String projectId, + final String datasetId, + final String tableId, + final JsonReader rows) throws IOException{ + + + final Bigquery bigquery = BigqueryServiceFactory.getService(); + final Gson gson = new Gson(); + rows.beginArray(); + + return new Iterator(){ + + public boolean hasNext() { + try { + return rows.hasNext(); + } catch (IOException e) { + // TODO(elibixby): Auto-generated catch block + e.printStackTrace(); + } + return false; + } + + public TableDataInsertAllResponse next() { + try { + Map rowData = gson.>fromJson( + rows, + (new HashMap()).getClass()); + return streamRow(bigquery, + projectId, + datasetId, + tableId, + new TableDataInsertAllRequest.Rows().setJson(rowData)); + } catch (JsonSyntaxException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public void remove() { + this.next(); + } + + }; + + } +// [END run] + +// [START streamRow] + public static TableDataInsertAllResponse streamRow(Bigquery bigquery, + String projectId, + String datasetId, + String tableId, + TableDataInsertAllRequest.Rows row) throws IOException{ + + return bigquery.tabledata().insertAll( + projectId, + datasetId, + tableId, + new TableDataInsertAllRequest().setRows(Collections.singletonList(row))).execute(); + + } +// [END streamRow] +} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java new file mode 100644 index 00000000000..8d644a2d2f6 --- /dev/null +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java @@ -0,0 +1,81 @@ +/* + Copyright 2015, Google, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.google.cloud.bigquery.samples; + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.QueryResponse; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Scanner; +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class SyncQuerySample extends BigqueryUtils{ + + + //[START main] + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) + throws IOException{ + + + Scanner scanner = new Scanner(System.in); + System.out.println("Enter your project id: "); + String projectId = scanner.nextLine(); + System.out.println("Enter your query string: "); + String queryString = scanner.nextLine(); + System.out.println("Enter how long to wait for the query to complete (in milliseconds):\n " + + "(if longer than 10 seconds, use an asynchronous query)"); + long waitTime = scanner.nextLong(); + scanner.close(); + Iterator pages = run(projectId, queryString, waitTime); + while(pages.hasNext()){ + printRows(pages.next().getRows(), System.out); + } + } + // [END main] + + + // [START run] + public static Iterator run(String projectId, + String queryString, + long waitTime) throws IOException{ + Bigquery bigquery = BigqueryServiceFactory.getService(); + //Wait until query is done with 10 second timeout, at most 5 retries on error + QueryResponse query = bigquery.jobs().query( + projectId, + new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString)).execute(); + + //Make a request to get the results of the query + //(timeout is zero since job should be complete) + + GetQueryResults getRequest = bigquery.jobs().getQueryResults( + query.getJobReference().getProjectId(), + query.getJobReference().getJobId()); + + + return getPages(getRequest); + } + // [END run] + + +} diff --git a/bigquery/src/main/resources/constants.json b/bigquery/src/main/resources/constants.json new file mode 100644 index 00000000000..1a2eb4b69f2 --- /dev/null +++ b/bigquery/src/main/resources/constants.json @@ -0,0 +1,9 @@ +{ + "projectId": "bigquery-devrel-samples", + "datasetId": "test_dataset", + "currentTableId": "test_table", + "newTableId": "test_table2", + "cloudStorageInputURI": "gs://bigquery-devrel-samples-bucket/data.csv", + "cloudStorageOutputURI": "gs://bigquery-devrel-samples-bucket/output.csv", + "query": "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;" +} diff --git a/bigquery/src/main/resources/schema.json b/bigquery/src/main/resources/schema.json new file mode 100644 index 00000000000..b61fd26be03 --- /dev/null +++ b/bigquery/src/main/resources/schema.json @@ -0,0 +1 @@ +[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] diff --git a/bigquery/src/main/resources/streamrows.json b/bigquery/src/main/resources/streamrows.json new file mode 100644 index 00000000000..c65f6291054 --- /dev/null +++ b/bigquery/src/main/resources/streamrows.json @@ -0,0 +1,7 @@ +[ + {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 1, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 2, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 3, "Weight": 100.0, "IsMagic": false}, + {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false} +] diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java new file mode 100644 index 00000000000..49a6735bf94 --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java @@ -0,0 +1,49 @@ +package com.google.cloud.bigquery.samples.test; + +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.cloud.bigquery.samples.AsyncQuerySample; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +import org.junit.*; + +import static org.junit.Assert.*; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Iterator; + +public class AsyncQuerySampleTest extends BigquerySampleTest{ + + /** + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws FileNotFoundException + */ + public AsyncQuerySampleTest() throws JsonSyntaxException, JsonIOException, + FileNotFoundException { + super(); + // TODO(elibixby): Auto-generated constructor stub + } + + + @Test + public void testInteractive() throws IOException, InterruptedException{ + Iterator pages = AsyncQuerySample.run(CONSTANTS.getProjectId(), CONSTANTS.getQuery(), false, 5000); + while(pages.hasNext()){ + assertTrue(!pages.next().getRows().isEmpty()); + } + } + + + @Test + @Ignore // Batches can take up to 3 hours to run, probably shouldn't use this + public void testBatch() throws IOException, InterruptedException{ + Iterator pages = AsyncQuerySample.run(CONSTANTS.getProjectId(), CONSTANTS.getQuery(), true, 5000); + while(pages.hasNext()){ + assertTrue(!pages.next().getRows().isEmpty()); + } + } + + +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java new file mode 100644 index 00000000000..80bb05ae491 --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/BigquerySampleTest.java @@ -0,0 +1,84 @@ +package com.google.cloud.bigquery.samples.test; + +import com.google.cloud.bigquery.samples.BigqueryUtils; +import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; + + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class BigquerySampleTest extends BigqueryUtils { + + protected static class Constants{ + private String projectId; + private String datasetId; + private String currentTableId; + private String newTableId; + private String cloudStorageInputURI; + private String cloudStorageOutputURI; + private String query; + /** + * @return the projectId + */ + public String getProjectId() { + return projectId; + } + /** + * @return the datasetId + */ + public String getDatasetId() { + return datasetId; + } + /** + * @return the currentTableId + */ + public String getCurrentTableId() { + return currentTableId; + } + /** + * @return the newTableId + */ + public String getNewTableId() { + return newTableId; + } + /** + * @return the query + */ + public String getQuery() { + return query; + } + /** + * @return the cloudStorageOutputURI + */ + public String getCloudStorageOutputURI() { + return cloudStorageOutputURI; + } + /** + * @return the cloudStorageInputURI + */ + public String getCloudStorageInputURI() { + return cloudStorageInputURI; + } + } + + protected static Constants CONSTANTS = null ; + + protected BigquerySampleTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException{ + if(CONSTANTS == null){ + InputStream is = this.getClass().getResourceAsStream + ("/constants.json"); + CONSTANTS = (new Gson()).fromJson( + new InputStreamReader(is), + Constants.class); + } + } + + + +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java new file mode 100644 index 00000000000..0c43f7e32c5 --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java @@ -0,0 +1,38 @@ +package com.google.cloud.bigquery.samples.test; + +import com.google.cloud.bigquery.samples.ExportDataCloudStorageSample; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class ExportDataCloudStorageSampleTest extends BigquerySampleTest { + + /** + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws FileNotFoundException + */ + public ExportDataCloudStorageSampleTest() throws JsonSyntaxException, JsonIOException, + FileNotFoundException { + super(); + } + + @Test + public void testExportData() throws IOException, InterruptedException{ + ExportDataCloudStorageSample.run(CONSTANTS.getCloudStorageOutputURI(), + CONSTANTS.getProjectId(), + CONSTANTS.getDatasetId(), + CONSTANTS.getCurrentTableId(), + 5000L); + } + + + +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCSVSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCSVSampleTest.java new file mode 100644 index 00000000000..109d6f7b77e --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCSVSampleTest.java @@ -0,0 +1,41 @@ +package com.google.cloud.bigquery.samples.test; + +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; +import com.google.cloud.bigquery.samples.LoadDataCSVSample; + +import org.junit.Test; + +import java.io.*; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class LoadDataCSVSampleTest extends BigquerySampleTest { + + /** + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws FileNotFoundException + */ + public LoadDataCSVSampleTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException { + // TODO(elibixby): Auto-generated constructor stub + super(); + } + + @Test + public void testLoadData() throws IOException, InterruptedException{ + + InputStreamReader is = new InputStreamReader(LoadDataCSVSample + .class.getResourceAsStream("/schema.json")); + LoadDataCSVSample.run( + CONSTANTS.getCloudStorageInputURI(), + CONSTANTS.getProjectId(), + CONSTANTS.getDatasetId(), + CONSTANTS.getNewTableId(), + is , + 5000L); + } + + +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java new file mode 100644 index 00000000000..4e1296a5884 --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java @@ -0,0 +1,47 @@ +package com.google.cloud.bigquery.samples.test; + +import static org.junit.Assert.*; + +import com.google.api.services.bigquery.model.TableDataInsertAllResponse; +import com.google.cloud.bigquery.samples.StreamingSample; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; +import com.google.gson.stream.JsonReader; + +import org.junit.Test; + +import java.io.*; +import java.util.Iterator; + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class StreamingSampleTest extends BigquerySampleTest { + + /** + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws FileNotFoundException + */ + public StreamingSampleTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException { + super(); + } + + @Test + public void testStream() throws IOException{ + JsonReader json = new JsonReader( + new InputStreamReader(BigquerySampleTest + .class.getResourceAsStream("/streamrows.json"))); + Iterator response = StreamingSample.run( + CONSTANTS.getProjectId(), + CONSTANTS.getDatasetId(), + CONSTANTS.getCurrentTableId(), + json); + + while(response.hasNext()){ + assertTrue(!response.next().isEmpty()); + } + + } + +} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java new file mode 100644 index 00000000000..c5bee385d4d --- /dev/null +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java @@ -0,0 +1,43 @@ +package com.google.cloud.bigquery.samples.test; + +import static org.junit.Assert.*; + +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.cloud.bigquery.samples.SyncQuerySample; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Iterator; + + +/** + * TODO: Insert description here. (generated by elibixby) + */ +public class SyncQuerySampleTest extends BigquerySampleTest { + + /** + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws FileNotFoundException + */ + public SyncQuerySampleTest() throws JsonSyntaxException, JsonIOException, + FileNotFoundException { + super(); + } + + @Test + public void testSyncQuery() throws IOException{ + Iterator pages = SyncQuerySample.run( + CONSTANTS.getProjectId(), + CONSTANTS.getQuery(), + 10000); + while(pages.hasNext()){ + assertTrue(!pages.next().getRows().isEmpty()); + } + } + +} diff --git a/pom.xml b/pom.xml index 4b1b4607d9a..f55c41f4af7 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ taskqueue/deferred unittests + bigquery @@ -68,11 +69,6 @@ ${appengine.version} test - - com.google.appengine - appengine-api-1.0-sdk - ${appengine.target.version} -