diff --git a/bigquery/README.md b/bigquery/README.md index 341ba38176b..ba73bf65ae2 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -18,8 +18,6 @@ recommended way to access the API. way to interact with BigQuery. - rest - This uses BigQuery's RESTful API directly. Not recommended. -- src - - This uses [Google API Client Libraries](https://developers.google.com/api-client-library/java/). Not recommended. ## Quickstart diff --git a/bigquery/cloud-client/README.md b/bigquery/cloud-client/README.md index d434fca760c..a8dd6cf6702 100644 --- a/bigquery/cloud-client/README.md +++ b/bigquery/cloud-client/README.md @@ -26,11 +26,12 @@ You can then run a given `ClassName` via: mvn exec:java -Dexec.mainClass=com.example.bigquery.QuickstartSample -### Running a synchronous query +### Running a query using standard SQL syntax - mvn exec:java -Dexec.mainClass=com.example.bigquery.SyncQuerySample \ - -Dquery='SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;' \ - -DuseLegacySql=false + mvn exec:java -Dexec.mainClass=com.example.bigquery.QuerySample \ + -Dexec.args=' \ + --query="SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;" \ + --runStandardSqlQuery' ### Running the simple app example diff --git a/bigquery/cloud-client/pom.xml b/bigquery/cloud-client/pom.xml index ccf87f68a2f..a8854a9429c 100644 --- a/bigquery/cloud-client/pom.xml +++ b/bigquery/cloud-client/pom.xml @@ -39,6 +39,11 @@ google-cloud-bigquery 0.12.0-beta + + commons-cli + commons-cli + 1.3.1 + joda-time joda-time diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java index 97f08a563c2..2213657cc39 100644 --- a/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java @@ -33,9 +33,7 @@ import java.util.Iterator; import java.util.List; -/** - * A sample that demonstrates use of query parameters. - */ +/** A sample that demonstrates use of query parameters. */ public class QueryParametersSample { private static final int ERROR_CODE = 1; @@ -43,22 +41,19 @@ private static void printUsage() { System.err.println("Usage:"); System.err.printf( "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", - QueryParametersSample.class.getCanonicalName(), - "${sample}"); + QueryParametersSample.class.getCanonicalName(), "${sample}"); System.err.println(); System.err.println("${sample} can be one of: named, array, timestamp"); System.err.println(); System.err.println("Usage for ${sample}=named:"); System.err.printf( "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", - QueryParametersSample.class.getCanonicalName(), - "named ${corpus} ${minWordCount}"); + QueryParametersSample.class.getCanonicalName(), "named ${corpus} ${minWordCount}"); System.err.println(); System.err.println("Usage for sample=array:"); System.err.printf( "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", - QueryParametersSample.class.getCanonicalName(), - "array ${gender} ${states...}"); + QueryParametersSample.class.getCanonicalName(), "array ${gender} ${states...}"); System.err.println(); System.err.println("\twhere ${gender} can be on of: M, F"); System.err.println( @@ -66,13 +61,10 @@ private static void printUsage() { System.err.println(); System.err.printf( "\t\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n", - QueryParametersSample.class.getCanonicalName(), - "array F MD WA"); + QueryParametersSample.class.getCanonicalName(), "array F MD WA"); } - /** - * Prompts the user for the required parameters to perform a query. - */ + /** Prompts the user for the required parameters to perform a query. */ public static void main(final String[] args) throws IOException, InterruptedException { if (args.length < 1) { System.err.println("Expected first argument 'sample'"); @@ -125,11 +117,12 @@ private static void runNamed(final String corpus, final long minWordCount) BigQuery bigquery = new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); - String queryString = "SELECT word, word_count\n" - + "FROM `bigquery-public-data.samples.shakespeare`\n" - + "WHERE corpus = @corpus\n" - + "AND word_count >= @min_word_count\n" - + "ORDER BY word_count DESC"; + String queryString = + "SELECT word, word_count\n" + + "FROM `bigquery-public-data.samples.shakespeare`\n" + + "WHERE corpus = @corpus\n" + + "AND word_count >= @min_word_count\n" + + "ORDER BY word_count DESC"; QueryRequest queryRequest = QueryRequest.newBuilder(queryString) .addNamedParameter("corpus", QueryParameterValue.string(corpus)) @@ -161,10 +154,7 @@ private static void runNamed(final String corpus, final long minWordCount) while (iter.hasNext()) { List row = iter.next(); - System.out.printf( - "%s: %d\n", - row.get(0).getStringValue(), - row.get(1).getLongValue()); + System.out.printf("%s: %d\n", row.get(0).getStringValue(), row.get(1).getLongValue()); } } // [END bigquery_query_params] @@ -173,24 +163,22 @@ private static void runNamed(final String corpus, final long minWordCount) * Query the baby names database to find the most popular names for a gender in a list of states. */ // [START bigquery_query_params_arrays] - private static void runArray(String gender, String[] states) - throws InterruptedException { + private static void runArray(String gender, String[] states) throws InterruptedException { BigQuery bigquery = new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); - String queryString = "SELECT name, sum(number) as count\n" - + "FROM `bigquery-public-data.usa_names.usa_1910_2013`\n" - + "WHERE gender = @gender\n" - + "AND state IN UNNEST(@states)\n" - + "GROUP BY name\n" - + "ORDER BY count DESC\n" - + "LIMIT 10;"; + String queryString = + "SELECT name, sum(number) as count\n" + + "FROM `bigquery-public-data.usa_names.usa_1910_2013`\n" + + "WHERE gender = @gender\n" + + "AND state IN UNNEST(@states)\n" + + "GROUP BY name\n" + + "ORDER BY count DESC\n" + + "LIMIT 10;"; QueryRequest queryRequest = QueryRequest.newBuilder(queryString) .addNamedParameter("gender", QueryParameterValue.string(gender)) - .addNamedParameter( - "states", - QueryParameterValue.array(states, String.class)) + .addNamedParameter("states", QueryParameterValue.array(states, String.class)) // Standard SQL syntax is required for parameterized queries. // See: https://cloud.google.com/bigquery/sql-reference/ .setUseLegacySql(false) @@ -272,8 +260,7 @@ private static void runTimestamp() throws InterruptedException { new DateTime( // Timestamp values are returned in microseconds since 1970-01-01T00:00:00 UTC, // but org.joda.time.DateTime constructor accepts times in milliseconds. - row.get(0).getTimestampValue() / 1000, - DateTimeZone.UTC))); + row.get(0).getTimestampValue() / 1000, DateTimeZone.UTC))); } } // [END bigquery_query_params_timestamps] diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/QuerySample.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/QuerySample.java new file mode 100644 index 00000000000..44d7b27e01b --- /dev/null +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery/QuerySample.java @@ -0,0 +1,208 @@ +/* + Copyright 2016, Google, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.example.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryResponse; +import com.google.cloud.bigquery.QueryResult; +import com.google.cloud.bigquery.TableId; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + +/** Runs a query against BigQuery. */ +public class QuerySample { + // [START query_config_simple] + public static void runSimpleQuery(String queryString) + throws TimeoutException, InterruptedException { + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(queryString).build(); + + runQuery(queryConfig); + } + // [END query_config_simple] + + // [START query_config_standard_sql] + public static void runStandardSqlQuery(String queryString) + throws TimeoutException, InterruptedException { + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(queryString) + // To use standard SQL syntax, set useLegacySql to false. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + runQuery(queryConfig); + } + // [END query_config_standard_sql] + + // [START query_config_permanent_table] + public static void runQueryPermanentTable( + String queryString, + String destinationDataset, + String destinationTable, + boolean allowLargeResults) throws TimeoutException, InterruptedException { + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(queryString) + // Save the results of the query to a permanent table. + // See: https://cloud.google.com/bigquery/querying-data#permanent-table + .setDestinationTable(TableId.of(destinationDataset, destinationTable)) + // Allow results larger than the maximum response size. + // If true, a destination table must be set. + // See: https://cloud.google.com/bigquery/querying-data#large-results + .setAllowLargeResults(allowLargeResults) + .build(); + + runQuery(queryConfig); + } + // [END query_config_permanent_table] + + // [START query_config_cache] + public static void runUncachedQuery(String queryString) + throws TimeoutException, InterruptedException { + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(queryString) + // Do not use the query cache. Force live query evaluation. + // See: https://cloud.google.com/bigquery/querying-data#query-caching + .setUseQueryCache(false) + .build(); + + runQuery(queryConfig); + } + // [END query_config_cache] + + // [START query_config_batch] + public static void runBatchQuery(String queryString) + throws TimeoutException, InterruptedException { + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(queryString) + // Run at batch priority, which won't count toward concurrent rate + // limit. + // See: https://cloud.google.com/bigquery/querying-data#interactive-batch + .setPriority(QueryJobConfiguration.Priority.BATCH) + .build(); + + runQuery(queryConfig); + } + // [END query_config_batch] + + + // [START run_query] + public static void runQuery(QueryJobConfiguration queryConfig) + throws TimeoutException, InterruptedException { + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + // Create a job ID so that we can safely retry. + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + // Wait for the query to complete. + queryJob = queryJob.waitFor(); + + // Check for errors + if (queryJob == null) { + throw new RuntimeException("Job no longer exists"); + } else if (queryJob.getStatus().getError() != null) { + // You can also look at queryJob.getStatus().getExecutionErrors() for all + // errors, not just the latest one. + throw new RuntimeException(queryJob.getStatus().getError().toString()); + } + + // Get the results. + QueryResponse response = bigquery.getQueryResults(jobId); + QueryResult result = response.getResult(); + + // Print all pages of the results. + while (result != null) { + if (response.hasErrors()) { + String firstError = ""; + if (response.getExecutionErrors().size() != 0) { + firstError = response.getExecutionErrors().get(0).getMessage(); + } + throw new RuntimeException(firstError); + } + + Iterator> iter = result.iterateAll(); + while (iter.hasNext()) { + List row = iter.next(); + for (FieldValue val : row) { + System.out.printf("%s,", val.toString()); + } + System.out.printf("\n"); + } + + result = result.getNextPage(); + } + } + // [END run_query] + + /** Prompts the user for the required parameters to perform a query. */ + public static void main(final String[] args) + throws IOException, InterruptedException, TimeoutException, ParseException { + Options options = new Options(); + + // Use an OptionsGroup to choose which sample to run. + OptionGroup samples = new OptionGroup(); + samples.addOption(Option.builder().longOpt("runSimpleQuery").build()); + samples.addOption(Option.builder().longOpt("runStandardSqlQuery").build()); + samples.addOption(Option.builder().longOpt("runPermanentTableQuery").build()); + samples.addOption(Option.builder().longOpt("runUncachedQuery").build()); + samples.addOption(Option.builder().longOpt("runBatchQuery").build()); + samples.isRequired(); + options.addOptionGroup(samples); + + options.addOption(Option.builder().longOpt("query").hasArg().required().build()); + options.addOption(Option.builder().longOpt("destDataset").hasArg().build()); + options.addOption(Option.builder().longOpt("destTable").hasArg().build()); + options.addOption(Option.builder().longOpt("allowLargeResults").build()); + + CommandLineParser parser = new DefaultParser(); + CommandLine cmd = parser.parse(options, args); + + String query = cmd.getOptionValue("query"); + if (cmd.hasOption("runSimpleQuery")) { + runSimpleQuery(query); + } else if (cmd.hasOption("runStandardSqlQuery")) { + runStandardSqlQuery(query); + } else if (cmd.hasOption("runPermanentTableQuery")) { + String destDataset = cmd.getOptionValue("destDataset"); + String destTable = cmd.getOptionValue("destTable"); + boolean allowLargeResults = cmd.hasOption("allowLargeResults"); + runQueryPermanentTable(query, destDataset, destTable, allowLargeResults); + } else if (cmd.hasOption("runUncachedQuery")) { + runUncachedQuery(query); + } else if (cmd.hasOption("runBatchQuery")) { + runBatchQuery(query); + } + } +} diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/SimpleApp.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/SimpleApp.java index b71fc3adfc2..5984a3957d1 100644 --- a/bigquery/cloud-client/src/main/java/com/example/bigquery/SimpleApp.java +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery/SimpleApp.java @@ -41,10 +41,10 @@ public static void main(String... args) throws Exception { // [START run_query] QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder( - "SELECT " - + "APPROX_TOP_COUNT(corpus, 10) as title, " - + "COUNT(*) as unique_words " - + "FROM `publicdata.samples.shakespeare`;") + "SELECT " + + "APPROX_TOP_COUNT(corpus, 10) as title, " + + "COUNT(*) as unique_words " + + "FROM `publicdata.samples.shakespeare`;") // Use standard SQL syntax for queries. // See: https://cloud.google.com/bigquery/sql-reference/ .setUseLegacySql(false) @@ -98,4 +98,3 @@ public static void main(String... args) throws Exception { } } // [END all] - diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery/SyncQuerySample.java b/bigquery/cloud-client/src/main/java/com/example/bigquery/SyncQuerySample.java deleted file mode 100644 index 94ccd8edece..00000000000 --- a/bigquery/cloud-client/src/main/java/com/example/bigquery/SyncQuerySample.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - Copyright 2016, Google, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package com.example.bigquery; - -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.FieldValue; -import com.google.cloud.bigquery.QueryRequest; -import com.google.cloud.bigquery.QueryResponse; -import com.google.cloud.bigquery.QueryResult; - -import java.io.IOException; -import java.io.PrintStream; -import java.util.Iterator; -import java.util.List; - -/** - * Runs a synchronous query against BigQuery. - */ -public class SyncQuerySample { - private static final String DEFAULT_QUERY = - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;"; - private static final long TEN_SECONDS_MILLIS = 10000; - - /** - * Prompts the user for the required parameters to perform a query. - */ - public static void main(final String[] args) throws IOException, InterruptedException { - String queryString = System.getProperty("query"); - if (queryString == null || queryString.isEmpty()) { - System.out.println("The query property was not set, using default."); - queryString = DEFAULT_QUERY; - } - System.out.printf("query: %s\n", queryString); - - String waitTimeString = System.getProperty("waitTime"); - if (waitTimeString == null || waitTimeString.isEmpty()) { - waitTimeString = "1000"; - } - long waitTime = Long.parseLong(waitTimeString); - System.out.printf("waitTime: %d (milliseconds)\n", waitTime); - if (waitTime > TEN_SECONDS_MILLIS) { - System.out.println( - "WARNING: If the query is going to take longer than 10 seconds to complete, use an" - + " asynchronous query."); - } - - String useLegacySqlString = System.getProperty("useLegacySql"); - if (useLegacySqlString == null || useLegacySqlString.isEmpty()) { - useLegacySqlString = "false"; - } - boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString); - - run(System.out, queryString, waitTime, useLegacySql); - } - - /** - * Perform the given query using the synchronous api. - * - * @param out stream to write results to - * @param queryString query to run - * @param waitTime Timeout in milliseconds before we abort - * @param useLegacySql Boolean that is false if using standard SQL syntax. - */ - // [START run] - public static void run( - final PrintStream out, - final String queryString, - final long waitTime, - final boolean useLegacySql) throws IOException, InterruptedException { - BigQuery bigquery = - new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance()); - - QueryRequest queryRequest = - QueryRequest.newBuilder(queryString) - .setMaxWaitTime(waitTime) - // Use standard SQL syntax or legacy SQL syntax for queries. - // See: https://cloud.google.com/bigquery/sql-reference/ - .setUseLegacySql(useLegacySql) - .build(); - QueryResponse response = bigquery.query(queryRequest); - - // Wait for the job to finish (if the query takes more than 10 seconds to complete). - while (!response.jobCompleted()) { - Thread.sleep(1000); - response = bigquery.getQueryResults(response.getJobId()); - } - - if (response.hasErrors()) { - String firstError = ""; - if (response.getExecutionErrors().size() != 0) { - firstError = response.getExecutionErrors().get(0).getMessage(); - } - throw new RuntimeException(firstError); - } - - QueryResult result = response.getResult(); - Iterator> iter = result.iterateAll(); - while (iter.hasNext()) { - List row = iter.next(); - for (FieldValue val : row) { - out.printf("%s,", val.toString()); - } - out.printf("\n"); - } - } - // [END run] -} diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java index 4577850d872..558667911fb 100644 --- a/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java +++ b/bigquery/cloud-client/src/test/java/com/example/bigquery/QueryParametersSampleIT.java @@ -27,9 +27,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; -/** - * Tests for simple app sample. - */ +/** Tests for simple app sample. */ @RunWith(JUnit4.class) @SuppressWarnings("checkstyle:abbreviationaswordinname") public class QueryParametersSampleIT { @@ -50,21 +48,21 @@ public void tearDown() { @Test public void testNamedSample() throws Exception { - QueryParametersSample.main(new String[]{"named", "romeoandjuliet", "100"}); + QueryParametersSample.main(new String[] {"named", "romeoandjuliet", "100"}); String got = bout.toString(); assertThat(got).contains("love"); } @Test public void testArraySample() throws Exception { - QueryParametersSample.main(new String[]{"array", "M", "WA", "WI", "WV", "WY"}); + QueryParametersSample.main(new String[] {"array", "M", "WA", "WI", "WV", "WY"}); String got = bout.toString(); assertThat(got).contains("James"); } @Test public void testTimestampSample() throws Exception { - QueryParametersSample.main(new String[]{"timestamp"}); + QueryParametersSample.main(new String[] {"timestamp"}); String got = bout.toString(); assertThat(got).contains("2016-12-07T09:00:00Z"); } diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/QuerySampleIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/QuerySampleIT.java new file mode 100644 index 00000000000..a0f7a007aa9 --- /dev/null +++ b/bigquery/cloud-client/src/test/java/com/example/bigquery/QuerySampleIT.java @@ -0,0 +1,128 @@ +/* + Copyright 2016, Google, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** Tests for query sample. */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class QuerySampleIT { + private static final String LEGACY_SQL_QUERY = + "SELECT corpus FROM [bigquery-public-data:samples.shakespeare] GROUP BY corpus;"; + private static final String STANDARD_SQL_QUERY = + "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; + private static final String CORPUS_NAME = "romeoandjuliet"; + private static final String TEST_DATASET = "query_sample_test"; + private static final String TEST_TABLE = "query_sample_test"; + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final void deleteTestDataset() { + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + DatasetId datasetId = DatasetId.of(TEST_DATASET); + BigQuery.DatasetDeleteOption deleteContents = BigQuery.DatasetDeleteOption.deleteContents(); + bigquery.delete(datasetId, deleteContents); + } + + private static final void createTestDataset() { + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + DatasetId datasetId = DatasetId.of(TEST_DATASET); + bigquery.create(DatasetInfo.newBuilder(datasetId).build()); + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testSimpleQuery() throws Exception { + QuerySample.main(new String[]{"--query", LEGACY_SQL_QUERY, "--runSimpleQuery"}); + String got = bout.toString(); + assertThat(got).contains(CORPUS_NAME); + } + + @Test + public void testStandardSqlQuery() throws Exception { + QuerySample.main(new String[]{"--query", STANDARD_SQL_QUERY, "--runStandardSqlQuery"}); + String got = bout.toString(); + assertThat(got).contains(CORPUS_NAME); + } + + @Test + public void testUncachedQuery() throws Exception { + QuerySample.main(new String[]{"--query", LEGACY_SQL_QUERY, "--runSimpleQuery"}); + String got = bout.toString(); + assertThat(got).contains(CORPUS_NAME); + } + + @Test + // Exclude this test from system tests. Batch queries usually start within a + // few minutes, but may not get scheduled for up to 24 hours. + // See: https://cloud.google.com/bigquery/querying-data#interactive-batch + @Ignore + public void testBatchQuery() throws Exception { + QuerySample.main(new String[]{"--query", LEGACY_SQL_QUERY, "--runBatchQuery"}); + String got = bout.toString(); + assertThat(got).contains(CORPUS_NAME); + } + + @Test + public void testPermanentTableQuery() throws Exception { + // Setup the test data. + deleteTestDataset(); + createTestDataset(); + + QuerySample.main( + new String[]{ + "--query", + LEGACY_SQL_QUERY, + "--runPermanentTableQuery", + "--destDataset", + TEST_DATASET, + "--destTable", + TEST_TABLE, + "--allowLargeResults"}); + String got = bout.toString(); + assertThat(got).contains(CORPUS_NAME); + + // Cleanup the test data. + deleteTestDataset(); + } +} diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/QuickstartSampleIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/QuickstartSampleIT.java index 25f2902eed3..297f5366195 100644 --- a/bigquery/cloud-client/src/test/java/com/example/bigquery/QuickstartSampleIT.java +++ b/bigquery/cloud-client/src/test/java/com/example/bigquery/QuickstartSampleIT.java @@ -31,9 +31,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; -/** - * Tests for quickstart sample. - */ +/** Tests for quickstart sample. */ @RunWith(JUnit4.class) @SuppressWarnings("checkstyle:abbreviationaswordinname") public class QuickstartSampleIT { diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/SimpleAppIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/SimpleAppIT.java index 689348d45a9..f4be05c2a95 100644 --- a/bigquery/cloud-client/src/test/java/com/example/bigquery/SimpleAppIT.java +++ b/bigquery/cloud-client/src/test/java/com/example/bigquery/SimpleAppIT.java @@ -27,9 +27,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; -/** - * Tests for simple app sample. - */ +/** Tests for simple app sample. */ @RunWith(JUnit4.class) @SuppressWarnings("checkstyle:abbreviationaswordinname") public class SimpleAppIT { diff --git a/bigquery/cloud-client/src/test/java/com/example/bigquery/SyncQuerySampleIT.java b/bigquery/cloud-client/src/test/java/com/example/bigquery/SyncQuerySampleIT.java deleted file mode 100644 index fff489c9e86..00000000000 --- a/bigquery/cloud-client/src/test/java/com/example/bigquery/SyncQuerySampleIT.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - Copyright 2016, Google, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package com.example.bigquery; - -import static com.google.common.truth.Truth.assertThat; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - -/** - * Tests for synchronous query sample. - */ -@RunWith(JUnit4.class) -@SuppressWarnings("checkstyle:abbreviationaswordinname") -public class SyncQuerySampleIT { - private ByteArrayOutputStream bout; - private PrintStream out; - - @Before - public void setUp() { - bout = new ByteArrayOutputStream(); - out = new PrintStream(bout); - } - - @Test - public void testSyncQuery() throws Exception { - SyncQuerySample.run( - out, - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;", - 10000 /* 10 seconds timeout */, - false /* useLegacySql */); - - String got = bout.toString(); - assertThat(got).contains("romeoandjuliet"); - } - - @Test - public void testSyncQueryLegacySql() throws Exception { - SyncQuerySample.run( - out, - "SELECT corpus FROM [publicdata:samples.shakespeare] GROUP BY corpus;", - 10000 /* 10 seconds timeout */, - true /* useLegacySql */); - - String got = bout.toString(); - assertThat(got).contains("romeoandjuliet"); - } -} diff --git a/bigquery/pom.xml b/bigquery/pom.xml deleted file mode 100644 index 0e16e3f71eb..00000000000 --- a/bigquery/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - 4.0.0 - com.google.cloud.bigquery.samples - bigquery-samples - jar - - - doc-samples - com.google.cloud - 1.0.0 - .. - - - - - googleapis - https://google-api-client-libraries.appspot.com/mavenrepo - - - - - - com.google.apis - google-api-services-bigquery - v2-rev343-1.22.0 - - - com.google.guava - guava-jdk5 - - - - - com.google.guava - guava - 20.0 - - - com.google.oauth-client - google-oauth-client - 1.22.0 - - - com.google.http-client - google-http-client-jackson2 - 1.22.0 - - - com.google.oauth-client - google-oauth-client-jetty - 1.22.0 - - - com.google.code.gson - gson - 2.8.0 - - - junit - junit - 4.12 - test - - - com.google.truth - truth - 0.32 - test - - - - - UTF-8 - - - - src/main/java - - - src/main/resources - - - - - diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java deleted file mode 100644 index e0384da0867..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.bigquery.samples; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationQuery; - -import java.io.IOException; -import java.util.Iterator; - -/** - * Example of authorizing with BigQuery and reading from a public dataset. - */ -public class AsyncQuerySample { - private static final String DEFAULT_QUERY = - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;"; - - // [START main] - /** - * Prompts for all the parameters required to make a query. - * - * @param args Command line args - * @throws IOException IOException - * @throws InterruptedException InterruptedException - */ - public static void main(final String[] args) throws IOException, InterruptedException { - String projectId = System.getProperty("projectId"); - if (projectId == null || projectId.isEmpty()) { - System.err.println("The projectId property must be set."); - System.exit(1); - } - System.out.printf("projectId: %s\n", projectId); - - String queryString = System.getProperty("query"); - if (queryString == null || queryString.isEmpty()) { - System.out.println("The query property was not set, using default."); - queryString = DEFAULT_QUERY; - } - System.out.printf("query: %s\n", queryString); - - String useBatchString = System.getProperty("useBatchMode"); - if (useBatchString == null || useBatchString.isEmpty()) { - useBatchString = "false"; - } - boolean useBatchMode = Boolean.parseBoolean(useBatchString); - System.out.printf("useBatchMode: %b\n", useBatchMode); - - String waitTimeString = System.getProperty("waitTime"); - if (waitTimeString == null || waitTimeString.isEmpty()) { - waitTimeString = "1000"; - } - long waitTime = Long.parseLong(waitTimeString); - System.out.printf("waitTime: %d (milliseconds)\n", waitTime); - - String useLegacySqlString = System.getProperty("useLegacySql"); - if (useLegacySqlString == null || useLegacySqlString.isEmpty()) { - useLegacySqlString = "false"; - } - boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString); - - Iterator pages = - run(projectId, queryString, useBatchMode, waitTime, useLegacySql); - while (pages.hasNext()) { - BigQueryUtils.printRows(pages.next().getRows(), System.out); - } - } - // [END main] - - // [START run] - /** - * Run the query. - * - * @param projectId Get this from Google Developers console - * @param queryString Query we want to run against BigQuery - * @param useBatchMode True if you want to batch the queries - * @param waitTime How long to wait before retries - * @param useLegacySql Boolean that is false if using standard SQL syntax. - * @return An iterator to the result of your pages - * @throws IOException Thrown if there's an IOException - * @throws InterruptedException Thrown if there's an Interrupted Exception - */ - public static Iterator run( - final String projectId, - final String queryString, - final boolean useBatchMode, - final long waitTime, - final boolean useLegacySql) - throws IOException, InterruptedException { - - Bigquery bigquery = BigQueryServiceFactory.getService(); - - Job query = asyncQuery(bigquery, projectId, queryString, useBatchMode, useLegacySql); - Bigquery.Jobs.Get getRequest = - bigquery.jobs().get(projectId, query.getJobReference().getJobId()); - - // Poll every waitTime milliseconds, - // retrying at most retries times if there are errors - BigQueryUtils.pollJob(getRequest, waitTime); - - GetQueryResults resultsRequest = - bigquery.jobs().getQueryResults(projectId, query.getJobReference().getJobId()); - - return BigQueryUtils.getPages(resultsRequest); - } - // [END run] - - // [START asyncQuery] - /** - * Inserts an asynchronous query Job for a particular query. - * - * @param bigquery an authorized BigQuery client - * @param projectId a String containing the project ID - * @param querySql the actual query string - * @param useBatchMode True if you want to run the query as BATCH - * @param useLegacySql Boolean that is false if using standard SQL syntax. - * @return a reference to the inserted query job - * @throws IOException Thrown if there's a network exception - */ - public static Job asyncQuery( - final Bigquery bigquery, - final String projectId, - final String querySql, - final boolean useBatchMode, - final boolean useLegacySql) - throws IOException { - - JobConfigurationQuery queryConfig = - new JobConfigurationQuery() - .setQuery(querySql) - // Set the useLegacySql parameter to false to use standard SQL syntax. See: - // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql - .setUseLegacySql(useLegacySql); - - if (useBatchMode) { - queryConfig.setPriority("BATCH"); - } - - Job job = new Job().setConfiguration(new JobConfiguration().setQuery(queryConfig)); - - return bigquery.jobs().insert(projectId, job).execute(); - } - // [END asyncQuery] - -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java deleted file mode 100644 index 9d6daee7185..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryServiceFactory.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples; - -// [START imports] -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.BigqueryScopes; -// [END imports] - -import java.io.IOException; -import java.util.Collection; - -/** - * This class creates our Service to connect to BigQuery including auth. - */ -public final class BigQueryServiceFactory { - - /** - * Private constructor to disable creation of this utility Factory class. - */ - private BigQueryServiceFactory() {} - - /** - * Singleton service used through the app. - */ - private static Bigquery service = null; - - /** - * Mutex created to create the singleton in thread-safe fashion. - */ - private static Object serviceLock = new Object(); - - /** - * Threadsafe Factory that provides an authorized BigQuery service. - * @return The BigQuery service - * @throws IOException Throw if there is an error connecting to BigQuery. - */ - public static Bigquery getService() throws IOException { - if (service == null) { - synchronized (serviceLock) { - if (service == null) { - service = createAuthorizedClient(); - } - } - } - return service; - } - - /** - * Creates an authorized client to Google BigQuery. - * - * @return The BigQuery Service - * @throws IOException Thrown if there is an error connecting - */ - // [START get_service] - private static Bigquery createAuthorizedClient() throws IOException { - // Create the credential - HttpTransport transport = new NetHttpTransport(); - JsonFactory jsonFactory = new JacksonFactory(); - GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); - - // Depending on the environment that provides the default credentials (e.g. Compute Engine, App - // Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the BigQuery scope if required. - if (credential.createScopedRequired()) { - Collection bigqueryScopes = BigqueryScopes.all(); - credential = credential.createScoped(bigqueryScopes); - } - - return new Bigquery.Builder(transport, jsonFactory, credential) - .setApplicationName("BigQuery Samples") - .build(); - } - // [END get_service] - -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java deleted file mode 100644 index 33f9fb247d6..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/BigQueryUtils.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright 2015, Google, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package com.google.cloud.bigquery.samples; - -import com.google.api.client.json.GenericJson; -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.Bigquery.Datasets; -import com.google.api.services.bigquery.BigqueryRequest; -import com.google.api.services.bigquery.model.DatasetList; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.TableCell; -import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TableSchema; -import com.google.gson.Gson; - -import java.io.IOException; -import java.io.PrintStream; -import java.io.Reader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -/** - * Helper functions for the other classes. - */ -public class BigQueryUtils { - - /** - * Private constructor to prevent creation of this class, which is just all - * static helper methods. - */ - private BigQueryUtils() {} - - /** - * Print rows to the output stream in a formatted way. - * @param rows rows in bigquery - * @param out Output stream we want to print to - */ - // [START print_rows] - public static void printRows(final List rows, final PrintStream out) { - for (TableRow row : rows) { - for (TableCell field : row.getF()) { - out.printf("%-50s", field.getV()); - } - out.println(); - } - } - // [END print_rows] - - /** - * Polls the job for completion. - * @param request The bigquery request to poll for completion - * @param interval Number of milliseconds between each poll - * @return The finished job - * @throws IOException IOException - * @throws InterruptedException InterruptedException - */ - // [START poll_job] - public static Job pollJob(final Bigquery.Jobs.Get request, final long interval) - throws IOException, InterruptedException { - Job job = request.execute(); - while (!job.getStatus().getState().equals("DONE")) { - System.out.println( - "Job is " + job.getStatus().getState() + " waiting " + interval + " milliseconds..."); - Thread.sleep(interval); - job = request.execute(); - } - return job; - } - // [END poll_job] - - /** - * Pages through the results of an arbitrary Bigquery request. - * @param requestTemplate The object that represents the call to fetch - * the results. - * @param The type of the returned objects - * @return An iterator that pages through the returned object - */ - // [START paging] - public static Iterator getPages( - final BigqueryRequest requestTemplate) { - - /** - * An iterator class that pages through a Bigquery request. - */ - class PageIterator implements Iterator { - - private BigqueryRequest request; - private boolean hasNext = true; - - /** - * Inner class that represents our iterator to page through results. - * @param requestTemplate The object that represents the call to fetch - * the results. - */ - public PageIterator(final BigqueryRequest requestTemplate) { - this.request = requestTemplate; - } - - /** - * Checks whether there is another page of results. - * @return True if there is another page of results. - */ - public boolean hasNext() { - return hasNext; - } - - /** - * Returns the next page of results. - * @return The next page of resul.ts - */ - public T next() { - if (!hasNext) { - throw new NoSuchElementException(); - } - try { - T response = request.execute(); - if (response.containsKey("pageToken")) { - request = request.set("pageToken", response.get("pageToken")); - } else { - hasNext = false; - } - return response; - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - /** - * Skips the page by moving the iterator to the next page. - */ - public void remove() { - this.next(); - } - } - - return new PageIterator(requestTemplate); - } - // [END paging] - - /** - * Loads a Bigquery schema. - * @param schemaSource The source of the schema - * @return The TableSchema - */ - // [START load_schema] - public static TableSchema loadSchema(final Reader schemaSource) { - TableSchema sourceSchema = new TableSchema(); - - List fields = - (new Gson()) - .>fromJson( - schemaSource, (new ArrayList()).getClass()); - - sourceSchema.setFields(fields); - - return sourceSchema; - } - // [END load_schema] - - // [START list_datasets] - /** - * Display all BigQuery datasets associated with a project. - * - * @param bigquery an authorized BigQuery client - * @param projectId a string containing the current project ID - * @throws IOException Thrown if there is a network error connecting to - * Bigquery. - */ - public static void listDatasets(final Bigquery bigquery, final String projectId) - throws IOException { - Datasets.List datasetRequest = bigquery.datasets().list(projectId); - DatasetList datasetList = datasetRequest.execute(); - if (datasetList.getDatasets() != null) { - List datasets = datasetList.getDatasets(); - System.out.println("Available datasets\n----------------"); - System.out.println(datasets.toString()); - for (DatasetList.Datasets dataset : datasets) { - System.out.format("%s\n", dataset.getDatasetReference().getDatasetId()); - } - } - } - // [END list_datasets] -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java deleted file mode 100644 index 03316dddc05..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ExportDataCloudStorageSample.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - Copyright 2015, Google, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.google.cloud.bigquery.samples; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationExtract; -import com.google.api.services.bigquery.model.TableReference; - -import java.io.IOException; -import java.util.Scanner; - -/** - * Sample of how to Export Cloud Data. - */ -public class ExportDataCloudStorageSample { - /** - * Protected constructor since this is a collection of static functions. - */ - protected ExportDataCloudStorageSample() { - super(); - } - - /** - * This program can be run to demonstrate running a Bigquery query from the - * CLI. - * @param args Command line args - * @throws IOException If there is an error connceting to bigquery - * @throws InterruptedException Should never be thrown. - */ - // [START main] - public static void main(final String[] args) throws IOException, InterruptedException { - Scanner scanner = new Scanner(System.in); - System.out.println("Enter your project id: "); - String projectId = scanner.nextLine(); - System.out.println("Enter your dataset id: "); - String datasetId = scanner.nextLine(); - System.out.println("Enter your table id: "); - String tableId = scanner.nextLine(); - System.out.println("Enter the Google Cloud Storage Path to which you'd " + "like to export: "); - String cloudStoragePath = scanner.nextLine(); - System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); - long interval = scanner.nextLong(); - scanner.close(); - - run(cloudStoragePath, projectId, datasetId, tableId, interval); - } - // [END main] - - /** - * Run the bigquery ClI. - * @param cloudStoragePath The bucket we are using - * @param projectId Project id - * @param datasetId datasetid - * @param tableId tableid - * @param interval interval to wait between polling in milliseconds - * @throws IOException Thrown if there is an error connecting to Bigquery. - * @throws InterruptedException Should never be thrown - */ - // [START run] - public static void run( - final String cloudStoragePath, - final String projectId, - final String datasetId, - final String tableId, - final long interval) - throws IOException, InterruptedException { - - Bigquery bigquery = BigQueryServiceFactory.getService(); - - Job extractJob = - extractJob( - bigquery, - cloudStoragePath, - new TableReference() - .setProjectId(projectId) - .setDatasetId(datasetId) - .setTableId(tableId)); - - Bigquery.Jobs.Get getJob = - bigquery - .jobs() - .get( - extractJob.getJobReference().getProjectId(), - extractJob.getJobReference().getJobId()); - - BigQueryUtils.pollJob(getJob, interval); - - System.out.println("Export is Done!"); - } - // [END run] - - /** - * A job that extracts data from a table. - * @param bigquery Bigquery service to use - * @param cloudStoragePath Cloud storage bucket we are inserting into - * @param table Table to extract from - * @return The job to extract data from the table - * @throws IOException Thrown if error connceting to Bigtable - */ - // [START extract_job] - public static Job extractJob( - final Bigquery bigquery, final String cloudStoragePath, final TableReference table) - throws IOException { - - JobConfigurationExtract extract = - new JobConfigurationExtract().setSourceTable(table).setDestinationUri(cloudStoragePath); - - return bigquery - .jobs() - .insert( - table.getProjectId(), - new Job().setConfiguration(new JobConfiguration().setExtract(extract))) - .execute(); - } - // [END extract_job] -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java deleted file mode 100644 index b4549b62240..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/GettingStarted.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2012 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples; - -// [START all] -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.BigqueryScopes; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.api.services.bigquery.model.QueryRequest; -import com.google.api.services.bigquery.model.QueryResponse; -import com.google.api.services.bigquery.model.TableCell; -import com.google.api.services.bigquery.model.TableRow; - -import java.io.IOException; -import java.util.List; -import java.util.Scanner; - -/** - * Example of authorizing with Bigquery and reading from a public dataset. - * - * Specifically, this queries the shakespeare dataset to fetch the 10 of Shakespeare's works with - * the greatest number of distinct words. - */ -public class GettingStarted { - // [START build_service] - /** - * Creates an authorized Bigquery client service using Application Default Credentials. - * - * @return an authorized Bigquery client - * @throws IOException if there's an error getting the default credentials. - */ - public static Bigquery createAuthorizedClient() throws IOException { - // Create the credential - HttpTransport transport = new NetHttpTransport(); - JsonFactory jsonFactory = new JacksonFactory(); - GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); - - // Depending on the environment that provides the default credentials (e.g. Compute Engine, App - // Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the Bigquery scope if required. - if (credential.createScopedRequired()) { - credential = credential.createScoped(BigqueryScopes.all()); - } - - return new Bigquery.Builder(transport, jsonFactory, credential) - .setApplicationName("Bigquery Samples") - .build(); - } - // [END build_service] - - // [START run_query] - /** - * Executes the given query synchronously. - * - * @param querySql the query to execute. - * @param bigquery the Bigquery service object. - * @param projectId the id of the project under which to run the query. - * @return a list of the results of the query. - * @throws IOException if there's an error communicating with the API. - */ - private static List executeQuery(String querySql, Bigquery bigquery, String projectId) - throws IOException { - QueryResponse query = - bigquery.jobs().query(projectId, new QueryRequest().setQuery(querySql)).execute(); - - // Execute it - GetQueryResultsResponse queryResult = - bigquery - .jobs() - .getQueryResults( - query.getJobReference().getProjectId(), query.getJobReference().getJobId()) - .execute(); - - return queryResult.getRows(); - } - // [END run_query] - - // [START print_results] - /** - * Prints the results to standard out. - * - * @param rows the rows to print. - */ - private static void printResults(List rows) { - System.out.print("\nQuery Results:\n------------\n"); - for (TableRow row : rows) { - for (TableCell field : row.getF()) { - System.out.printf("%-50s", field.getV()); - } - System.out.println(); - } - } - // [END print_results] - - /** - * Exercises the methods defined in this class. - * - * In particular, it creates an authorized Bigquery service object using Application Default - * Credentials, then executes a query against the public Shakespeare dataset and prints out the - * results. - * - * @param args the first argument, if it exists, should be the id of the project to run the test - * under. If no arguments are given, it will prompt for it. - * @throws IOException if there's an error communicating with the API. - */ - public static void main(String[] args) throws IOException { - Scanner sc; - if (args.length == 0) { - // Prompt the user to enter the id of the project to run the queries under - System.out.print("Enter the project ID: "); - sc = new Scanner(System.in); - } else { - sc = new Scanner(args[0]); - } - String projectId = sc.nextLine(); - - // Create a new Bigquery client authorized via Application Default Credentials. - Bigquery bigquery = createAuthorizedClient(); - - List rows = - executeQuery( - "SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words " - + "FROM [publicdata:samples.shakespeare]", - bigquery, - projectId); - - printResults(rows); - } -} -// [END all] diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java deleted file mode 100644 index 0334a2f2eda..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/ListDatasetsProjects.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples; - -import com.google.api.client.util.Data; -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.Bigquery.Datasets; -import com.google.api.services.bigquery.model.DatasetList; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.api.services.bigquery.model.ProjectList; -import com.google.api.services.bigquery.model.QueryRequest; -import com.google.api.services.bigquery.model.QueryResponse; -import com.google.api.services.bigquery.model.TableCell; -import com.google.api.services.bigquery.model.TableRow; - -import java.io.IOException; -import java.io.PrintStream; -import java.lang.Thread; -import java.util.List; - -/** - * Invokes the BigQuery basic APIs for the given project id specified. - * - * Samples used in this page: - * - * https://cloud.google.com/bigquery/bigquery-api-quickstart - */ -public class ListDatasetsProjects { - /** - * Run the sample. - */ - public static void main(String[] args) throws IOException, InterruptedException { - if (args.length != 1) { - System.err.println("Usage: QuickStart "); - return; - } - String projectId = args[0]; - - Bigquery bigquery = BigQueryServiceFactory.getService(); - String query = - "SELECT TOP( title, 10) as title, COUNT(*) as revision_count " - + "FROM [publicdata:samples.wikipedia] WHERE wp_namespace = 0;"; - - System.out.println(); - System.out.println("----- Running the asynchronous query and printing it to stdout."); - runQueryRpcAndPrint(bigquery, projectId, query, System.out); - - System.out.println(); - System.out.println("----- Listing all the Datasets in the projectId"); - listDatasets(bigquery, projectId); - - System.out.println(); - System.out.println("----- Listing all the Projects"); - listProjects(bigquery); - } - - /** - * Lists all Datasets in a project specified by the projectId. - * - * @param bigquery The BigQuery object. - * @param projectId The projectId from which lists the existing Datasets. - * @throws IOException if there's trouble with the network request. - */ - // [START listDatasets] - public static void listDatasets(Bigquery bigquery, String projectId) throws IOException { - Datasets.List datasetRequest = bigquery.datasets().list(projectId); - DatasetList datasetList = datasetRequest.execute(); - - if (datasetList.getDatasets() != null) { - List datasets = datasetList.getDatasets(); - System.out.println("Dataset list:"); - - for (DatasetList.Datasets dataset : datasets) { - System.out.format("%s\n", dataset.getDatasetReference().getDatasetId()); - } - } - } - // [END listDatasets] - - /** - * Lists all Projects. - * - * @param bigquery The BigQuery object. - * @throws IOException if there's trouble with the network request. - */ - // [START listProjects] - public static void listProjects(Bigquery bigquery) throws IOException { - Bigquery.Projects.List projectListRequest = bigquery.projects().list(); - ProjectList projectList = projectListRequest.execute(); - - if (projectList.getProjects() != null) { - List projects = projectList.getProjects(); - System.out.println("Project list:"); - - for (ProjectList.Projects project : projects) { - System.out.format("%s\n", project.getFriendlyName()); - } - } - } - // [END listProjects] - - /** - * Runs a synchronous BigQuery query and displays the result. - * - * @param bigquery An authorized BigQuery client - * @param projectId The current project id - * @param query A String containing a BigQuery SQL statement - * @param out A PrintStream for output, normally System.out - */ - static void runQueryRpcAndPrint( - Bigquery bigquery, String projectId, String query, PrintStream out) - throws IOException, InterruptedException { - QueryRequest queryRequest = new QueryRequest().setQuery(query); - QueryResponse queryResponse = bigquery.jobs().query(projectId, queryRequest).execute(); - if (queryResponse.getJobComplete()) { - printRows(queryResponse.getRows(), out); - if (null == queryResponse.getPageToken()) { - return; - } - } - // This loop polls until results are present, then loops over result pages. - String pageToken = null; - while (true) { - GetQueryResultsResponse queryResults = - bigquery - .jobs() - .getQueryResults(projectId, queryResponse.getJobReference().getJobId()) - .setPageToken(pageToken) - .execute(); - if (queryResults.getJobComplete()) { - printRows(queryResults.getRows(), out); - pageToken = queryResults.getPageToken(); - if (null == pageToken) { - return; - } - } - Thread.sleep(500); - } - } - - /** - * Print the given rows. - * - * @param rows the rows to print. - * @param out the place to print them. - */ - private static void printRows(java.util.List rows, PrintStream out) { - if (rows != null) { - for (TableRow row : rows) { - for (TableCell cell : row.getF()) { - // Data.isNull() is the recommended way to check for the 'null object' in TableCell. - out.printf("%s, ", Data.isNull(cell.getV()) ? "null" : cell.getV().toString()); - } - out.println(); - } - } - } -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java deleted file mode 100644 index b3ecb26c149..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/LoadDataCsvSample.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - Copyright 2015, Google, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package com.google.cloud.bigquery.samples; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.api.services.bigquery.model.TableReference; -import com.google.api.services.bigquery.model.TableSchema; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; -import java.util.Collections; -import java.util.Scanner; - -/** - * Cli tool to load data from a CSV into Bigquery. - */ -public class LoadDataCsvSample { - - /** - * Protected constructor since this is a collection of static methods. - */ - protected LoadDataCsvSample() {} - - /** - * Cli tool to load data from a CSV into Bigquery. - * @param args Command line args, should be empty - * @throws IOException IOException - * @throws InterruptedException InterruptedException - */ - // [START main] - public static void main(final String[] args) throws IOException, InterruptedException { - Scanner scanner = new Scanner(System.in); - System.out.println("Enter your project id: "); - String projectId = scanner.nextLine(); - System.out.println("Enter your dataset id: "); - String datasetId = scanner.nextLine(); - System.out.println("Enter your table id: "); - String tableId = scanner.nextLine(); - System.out.println("Enter the Google Cloud Storage Path to the data " + "you'd like to load: "); - String cloudStoragePath = scanner.nextLine(); - System.out.println("Enter the filepath to your schema: "); - String sourceSchemaPath = scanner.nextLine(); - - System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); - long interval = scanner.nextLong(); - scanner.close(); - - run( - cloudStoragePath, - projectId, - datasetId, - tableId, - new FileReader(new File(sourceSchemaPath)), - interval); - } - // [END main] - - /** - * Run the bigquery ClI. - * @param cloudStoragePath The bucket we are using - * @param projectId Project id - * @param datasetId datasetid - * @param tableId tableid - * @param schemaSource Source of the schema - * @param interval interval to wait between polling in milliseconds - * @throws IOException Thrown if there is an error connecting to Bigquery. - * @throws InterruptedException Should never be thrown - */ - // [START run] - public static void run( - final String cloudStoragePath, - final String projectId, - final String datasetId, - final String tableId, - final Reader schemaSource, - final long interval) - throws IOException, InterruptedException { - - Bigquery bigquery = BigQueryServiceFactory.getService(); - - Job loadJob = - loadJob( - bigquery, - cloudStoragePath, - new TableReference() - .setProjectId(projectId) - .setDatasetId(datasetId) - .setTableId(tableId), - BigQueryUtils.loadSchema(schemaSource)); - - Bigquery.Jobs.Get getJob = - bigquery - .jobs() - .get(loadJob.getJobReference().getProjectId(), loadJob.getJobReference().getJobId()); - - BigQueryUtils.pollJob(getJob, interval); - - System.out.println("Load is Done!"); - } - // [END run] - - /** - * A job that extracts data from a table. - * @param bigquery Bigquery service to use - * @param cloudStoragePath Cloud storage bucket we are inserting into - * @param table Table to extract from - * @param schema The schema of the table we are loading into - * @return The job to extract data from the table - * @throws IOException Thrown if error connceting to Bigtable - */ - // [START load_job] - public static Job loadJob( - final Bigquery bigquery, - final String cloudStoragePath, - final TableReference table, - final TableSchema schema) - throws IOException { - - JobConfigurationLoad load = - new JobConfigurationLoad() - .setDestinationTable(table) - .setSchema(schema) - .setSourceUris(Collections.singletonList(cloudStoragePath)); - - return bigquery - .jobs() - .insert( - table.getProjectId(), new Job().setConfiguration(new JobConfiguration().setLoad(load))) - .execute(); - } - // [END load_job] -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java deleted file mode 100644 index ef37f042aab..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/StreamingSample.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - Copyright 2015, Google, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.google.cloud.bigquery.samples; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.model.TableDataInsertAllRequest; -import com.google.api.services.bigquery.model.TableDataInsertAllResponse; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; -import com.google.gson.stream.JsonReader; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Scanner; - -/** - * Example of Bigquery Streaming. - */ -public class StreamingSample { - - /** - * Empty constructor since this is just a collection of static methods. - */ - protected StreamingSample() {} - - /** - * Command line that demonstrates Bigquery streaming. - * - * @param args Command line args, should be empty - * @throws IOException IOexception - */ - // [START main] - public static void main(final String[] args) throws IOException { - final Scanner scanner = new Scanner(System.in); - System.out.println("Enter your project id: "); - String projectId = scanner.nextLine(); - System.out.println("Enter your dataset id: "); - String datasetId = scanner.nextLine(); - System.out.println("Enter your table id: "); - String tableId = scanner.nextLine(); - scanner.close(); - - System.out.println( - "Enter JSON to stream to BigQuery: \n" + "Press End-of-stream (CTRL-D) to stop"); - - JsonReader fromCli = new JsonReader(new InputStreamReader(System.in)); - - Iterator responses = run(projectId, datasetId, tableId, fromCli); - - while (responses.hasNext()) { - System.out.println(responses.next()); - } - - fromCli.close(); - } - // [END main] - - /** - * Run the bigquery ClI. - * - * @param projectId Project id - * @param datasetId datasetid - * @param tableId tableid - * @param rows The source of the JSON rows we are streaming in. - * @return Returns Iterates through the stream responses - * @throws IOException Thrown if there is an error connecting to Bigquery. - * @throws InterruptedException Should never be thrown - */ - // [START run] - public static Iterator run( - final String projectId, final String datasetId, final String tableId, final JsonReader rows) - throws IOException { - - final Bigquery bigquery = BigQueryServiceFactory.getService(); - final Gson gson = new Gson(); - rows.beginArray(); - - return new Iterator() { - - /** - * Check whether there is another row to stream. - * - * @return True if there is another row in the stream - */ - public boolean hasNext() { - try { - return rows.hasNext(); - } catch (IOException e) { - e.printStackTrace(); - } - return false; - } - - /** - * Insert the next row, and return the response. - * - * @return Next page of data - */ - public TableDataInsertAllResponse next() { - try { - Map rowData = - gson.>fromJson(rows, (new HashMap()).getClass()); - return streamRow( - bigquery, - projectId, - datasetId, - tableId, - new TableDataInsertAllRequest.Rows().setJson(rowData)); - } catch (JsonSyntaxException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - public void remove() { - this.next(); - } - }; - } - // [END run] - - /** - * Stream the given row into the given bigquery table. - * - * @param bigquery The bigquery service - * @param projectId project id from Google Developers console - * @param datasetId id of the dataset - * @param tableId id of the table we're streaming - * @param row the row we're inserting - * @return Response from the insert - * @throws IOException ioexception - */ - // [START streamRow] - public static TableDataInsertAllResponse streamRow( - final Bigquery bigquery, - final String projectId, - final String datasetId, - final String tableId, - final TableDataInsertAllRequest.Rows row) - throws IOException { - - return bigquery - .tabledata() - .insertAll( - projectId, - datasetId, - tableId, - new TableDataInsertAllRequest().setRows(Collections.singletonList(row))) - .execute(); - } - // [END streamRow] -} diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java deleted file mode 100644 index bf3b0eb8bed..00000000000 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright 2015, Google, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.google.cloud.bigquery.samples; - -import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.api.services.bigquery.model.QueryRequest; -import com.google.api.services.bigquery.model.QueryResponse; - -import java.io.IOException; -import java.util.Iterator; - -/** - * Runs a synchronous query against Bigtable. - */ -public class SyncQuerySample { - private static final String DEFAULT_QUERY = - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;"; - private static final long TEN_SECONDS_MILLIS = 10000; - - /** - * Protected because this is a collection of static methods. - */ - protected SyncQuerySample() {} - - //[START main] - /** - * Prompts the user for the required parameters to perform a query. - * - * @param args args - * @throws IOException ioexceptino - */ - public static void main(final String[] args) throws IOException { - String projectId = System.getProperty("projectId"); - if (projectId == null || projectId.isEmpty()) { - System.err.println("The projectId property must be set."); - System.exit(1); - } - System.out.printf("projectId: %s\n", projectId); - - String queryString = System.getProperty("query"); - if (queryString == null || queryString.isEmpty()) { - System.out.println("The query property was not set, using default."); - queryString = DEFAULT_QUERY; - } - System.out.printf("query: %s\n", queryString); - - String waitTimeString = System.getProperty("waitTime"); - if (waitTimeString == null || waitTimeString.isEmpty()) { - waitTimeString = "1000"; - } - long waitTime = Long.parseLong(waitTimeString); - System.out.printf("waitTime: %d (milliseconds)\n", waitTime); - if (waitTime > TEN_SECONDS_MILLIS) { - System.out.println( - "WARNING: If the query is going to take longer than 10 seconds to complete, use an" - + " asynchronous query."); - } - - String useLegacySqlString = System.getProperty("useLegacySql"); - if (useLegacySqlString == null || useLegacySqlString.isEmpty()) { - useLegacySqlString = "false"; - } - boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString); - - Iterator pages = run(projectId, queryString, waitTime, useLegacySql); - while (pages.hasNext()) { - BigQueryUtils.printRows(pages.next().getRows(), System.out); - } - } - // [END main] - - /** - * Perform the given query using the synchronous api. - * - * @param projectId project id from developer console - * @param queryString query to run - * @param waitTime Timeout in milliseconds before we abort - * @param useLegacySql Boolean that is false if using standard SQL syntax. - * @return Iterator that pages through the results of the query - * @throws IOException ioexception - */ - // [START run] - public static Iterator run( - final String projectId, - final String queryString, - final long waitTime, - final boolean useLegacySql) throws IOException { - Bigquery bigquery = BigQueryServiceFactory.getService(); - - // Wait until query is done with `waitTime` millisecond timeout, at most 5 retries on error. - QueryResponse query = - bigquery - .jobs() - .query( - projectId, - new QueryRequest() - .setTimeoutMs(waitTime) - .setQuery(queryString) - // Set the useLegacySql parameter to false to use standard SQL syntax. See: - // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql - .setUseLegacySql(useLegacySql)) - .execute(); - - // Make a request to get the results of the query. - GetQueryResults getRequest = - bigquery - .jobs() - .getQueryResults( - query.getJobReference().getProjectId(), query.getJobReference().getJobId()); - - return BigQueryUtils.getPages(getRequest); - } - // [END run] - -} diff --git a/bigquery/src/main/resources/schema.json b/bigquery/src/main/resources/schema.json deleted file mode 100644 index b61fd26be03..00000000000 --- a/bigquery/src/main/resources/schema.json +++ /dev/null @@ -1 +0,0 @@ -[{"type": "STRING", "name": "Name"}, {"type": "INTEGER", "name": "Age"}, {"type": "FLOAT", "name": "Weight"}, {"type": "BOOLEAN", "name": "IsMagic"}] diff --git a/bigquery/src/main/resources/streamrows.json b/bigquery/src/main/resources/streamrows.json deleted file mode 100644 index c65f6291054..00000000000 --- a/bigquery/src/main/resources/streamrows.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 1, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 2, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 3, "Weight": 100.0, "IsMagic": false}, - {"Name": "test", "Age": 0, "Weight": 100.0, "IsMagic": false} -] diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java deleted file mode 100644 index a71e472acdb..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.cloud.bigquery.samples.AsyncQuerySample; - -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.IOException; -import java.util.Iterator; - -/** - * Tests for asynchronous query sample. - */ -@RunWith(JUnit4.class) -public class AsyncQuerySampleTest { - - @Test - public void testInteractive() throws IOException, InterruptedException { - Iterator pages = - AsyncQuerySample.run( - Constants.PROJECT_ID, - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;", - false /* useBatchMode */, - 5000, - false /* useLegacySql */); - while (pages.hasNext()) { - assertThat(pages.next().getRows()).isNotEmpty(); - } - } - - @Test - public void testInteractiveLegacySql() throws IOException, InterruptedException { - Iterator pages = - AsyncQuerySample.run( - Constants.PROJECT_ID, - Constants.QUERY, - false /* useBatchMode */, - 5000, - true /* useLegacySql */); - while (pages.hasNext()) { - assertThat(pages.next().getRows()).isNotEmpty(); - } - } - - @Test - @Ignore // Batches can take up to 3 hours to run, don't run during the system tests. - public void testBatch() throws IOException, InterruptedException { - Iterator pages = - AsyncQuerySample.run( - Constants.PROJECT_ID, - Constants.QUERY, - true /* useBatchMode */, - 5000, - true /* useLegacySql */); - while (pages.hasNext()) { - assertThat(pages.next().getRows()).isNotEmpty(); - } - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java deleted file mode 100644 index 4bddbc34bd1..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/Constants.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http=//www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -public class Constants { - public static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - public static final String BUCKET = PROJECT_ID; - public static final String DATASET_ID = "test_dataset_java"; - public static final String CURRENT_TABLE_ID = "test_table_java"; - public static final String NEW_TABLE_ID = "test_table_java_2"; - public static final String CLOUD_STORAGE_INPUT_URI = "gs://" + BUCKET + "/data.csv"; - public static final String CLOUD_STORAGE_OUTPUT_URI = "gs://" + BUCKET + "/output.csv"; - public static final String QUERY = - "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;"; -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java deleted file mode 100644 index c05e6b17eb0..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ExportDataCloudStorageSampleTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import com.google.cloud.bigquery.samples.ExportDataCloudStorageSample; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.IOException; - -/** - * Tests for export data Cloud Storage sample. - */ -@RunWith(JUnit4.class) -public class ExportDataCloudStorageSampleTest { - - @Test - public void testExportData() throws IOException, InterruptedException { - ExportDataCloudStorageSample.run( - Constants.CLOUD_STORAGE_OUTPUT_URI, - Constants.PROJECT_ID, - Constants.DATASET_ID, - Constants.CURRENT_TABLE_ID, - 5000L); - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java deleted file mode 100644 index 2a4babb4c1d..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/GettingStartedTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.bigquery.samples.GettingStarted; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; - -/** - * Test for GettingStarted.java - */ -@RunWith(JUnit4.class) -public class GettingStartedTest { - private static final PrintStream REAL_OUT = System.out; - private static final PrintStream REAL_ERR = System.err; - - private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); - private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); - - @Before - public void setUp() { - System.setOut(new PrintStream(stdout)); - System.setErr(new PrintStream(stderr)); - } - - @After - public void tearDown() { - System.setOut(REAL_OUT); - System.setErr(REAL_ERR); - } - - @Test - public void testSyncQuery() throws IOException { - GettingStarted.main(new String[] {Constants.PROJECT_ID}); - String out = stdout.toString(); - assertThat(out).named("stdout").containsMatch("Query Results:"); - assertThat(out).named("stdout").contains("hamlet"); - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java deleted file mode 100644 index 2bf05e7d3d0..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/ListDatasetsProjectsTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.bigquery.samples.ListDatasetsProjects; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.lang.Exception; - -/** - * Unit tests for {@link ListDatasetsProjects}. - */ -@RunWith(JUnit4.class) -public class ListDatasetsProjectsTest { - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - private static final PrintStream REAL_OUT = System.out; - private static final PrintStream REAL_ERR = System.err; - - private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); - private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); - - @Before - public void setUp() { - System.setOut(new PrintStream(stdout)); - System.setErr(new PrintStream(stderr)); - } - - @After - public void tearDown() { - System.setOut(REAL_OUT); - System.setErr(REAL_ERR); - } - - @Test - public void testUsage() throws Exception { - ListDatasetsProjects.main(new String[] {}); - assertThat(stderr.toString()).named("stderr").isEqualTo("Usage: QuickStart \n"); - } - - @Test - public void testMain() throws Exception { - ListDatasetsProjects.main(new String[] {Constants.PROJECT_ID}); - String out = stdout.toString(); - assertThat(out).named("stdout").contains("Running the asynchronous query"); - assertThat(out).named("stdout").containsMatch("George W. Bush, [0-9]+"); - assertThat(out).named("stdout").containsMatch("Wikipedia, [0-9]+"); - - assertThat(out).named("stdout").contains("Listing all the Datasets"); - assertThat(out).named("stdout").contains("test_dataset"); - - assertThat(out).named("stdout").contains("Listing all the Projects"); - assertThat(out).named("stdout").contains("Project list:"); - assertThat(out).named("stdout").containsMatch("Bigquery Samples|" + PROJECT_ID); - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java deleted file mode 100644 index c33201e71a4..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/LoadDataCsvSampleTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import com.google.cloud.bigquery.samples.LoadDataCsvSample; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.IOException; -import java.io.InputStreamReader; - -/** - * Tests for sample that loads data from CSV. - */ -@RunWith(JUnit4.class) -public class LoadDataCsvSampleTest { - - @Test - public void testLoadData() throws IOException, InterruptedException { - InputStreamReader is = - new InputStreamReader(LoadDataCsvSample.class.getResourceAsStream("/schema.json")); - LoadDataCsvSample.run( - Constants.CLOUD_STORAGE_INPUT_URI, - Constants.PROJECT_ID, - Constants.DATASET_ID, - Constants.NEW_TABLE_ID, - is, - 5000L); - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java deleted file mode 100644 index 01a6bd8f42a..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/StreamingSampleTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.services.bigquery.model.TableDataInsertAllResponse; -import com.google.cloud.bigquery.samples.StreamingSample; -import com.google.gson.stream.JsonReader; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Iterator; - -/** - * Tests for streaming sample. - */ -@RunWith(JUnit4.class) -public class StreamingSampleTest { - - @Test - public void testStream() throws IOException { - JsonReader json = - new JsonReader( - new InputStreamReader( - StreamingSampleTest.class.getResourceAsStream("/streamrows.json"))); - Iterator response = - StreamingSample.run( - Constants.PROJECT_ID, Constants.DATASET_ID, Constants.CURRENT_TABLE_ID, json); - - while (response.hasNext()) { - assertThat(response.next()).isNotEmpty(); - } - } -} diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java deleted file mode 100644 index e40d02f30f5..00000000000 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.bigquery.samples.test; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.cloud.bigquery.samples.SyncQuerySample; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -import java.io.IOException; -import java.util.Iterator; - -/** - * Tests for synchronous query sample. - */ -@RunWith(JUnit4.class) -public class SyncQuerySampleTest { - - @Test - public void testSyncQuery() throws IOException { - Iterator pages = - SyncQuerySample.run( - Constants.PROJECT_ID, - "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;", - 10000, - false /* useLegacySql */); - while (pages.hasNext()) { - assertThat(pages.next().getRows()).isNotEmpty(); - } - } - - @Test - public void testSyncQueryLegacySql() throws IOException { - Iterator pages = - SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000, true /* useLegacySql */); - while (pages.hasNext()) { - assertThat(pages.next().getRows()).isNotEmpty(); - } - } -} diff --git a/pom.xml b/pom.xml index 840a22a62fa..bba68f1af18 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,6 @@ flexible/static-files flexible/twilio - bigquery bigquery/cloud-client bigquery/rest compute/cmdline