Skip to content

Commit dd9a9be

Browse files
nirupa-kumarnirupa-kumar
authored
Language Automl samples (#126)
* intial Natural Language commit * fixed input path for skipped model test * removed DS store, attempting linter again * attempted linter fix * removed lodash * package json test directive fix, LLC fix * dependencies for mathjs, automl in sample file added * path fixes, project id fixes, test fixes * comment/test cleanup * fixed tutorial file * manual readme update * readme path fix model * mefailenglishthat'sunmpossible spelling fix * style fix for console statements * Style fixes; thanks Ace! * path fix! * Fix ENV variable for project Id (GCLOUD_PROJECT) * Language AutoML samples * fixing lint issues * Converting test to mocha * Checking if Kokoro failure was a blip
1 parent 8a4c29b commit dd9a9be

10 files changed

+1307
-0
lines changed

‎cloud-language/snippets/automl/.eslintrc.yml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
rules:
3+
no-console: off

‎cloud-language/snippets/automl/automlNaturalLanguageDataset.js

+393
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,393 @@
1+
/**
2+
* Copyright 2018, Google, LLC.
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
/**
17+
* This application demonstrates how to perform basic operations on dataset
18+
* with the Google AutoML Natural Language API.
19+
*
20+
* For more information, see the documentation at
21+
* https://cloud.google.com/natural-language/automl/docs/
22+
*/
23+
24+
`use strict`;
25+
26+
function createDataset(projectId, computeRegion, datasetName, multilabel) {
27+
// [START automl_natural_language_createDataset]
28+
const automl = require(`@google-cloud/automl`);
29+
30+
const client = new automl.v1beta1.AutoMlClient();
31+
32+
/**
33+
* TODO(developer): Uncomment the following line before running the sample.
34+
*/
35+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
36+
// const computeRegion = `region-name, e.g. "us-central1"`;
37+
// const datasetName = `name of the dataset to create, e.g. “myDataset”`;
38+
// const multiLabel = `type of the classification problem, e.g “False”, “True” (multilabel)`;
39+
40+
// A resource that represents Google Cloud Platform location.
41+
const projectLocation = client.locationPath(projectId, computeRegion);
42+
43+
// Classification type is assigned based on multilabel value.
44+
let classificationType = `MULTICLASS`;
45+
if (multilabel) {
46+
classificationType = `MULTILABEL`;
47+
}
48+
49+
// Set dataset name and metadata.
50+
const myDataset = {
51+
displayName: datasetName,
52+
textClassificationDatasetMetadata: {
53+
classificationType: classificationType,
54+
},
55+
};
56+
57+
// Create a dataset with the dataset metadata in the region.
58+
client
59+
.createDataset({parent: projectLocation, dataset: myDataset})
60+
.then(responses => {
61+
const dataset = responses[0];
62+
63+
// Display the dataset information.
64+
console.log(`Dataset name: ${dataset.name}`);
65+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
66+
console.log(`Dataset display name: ${dataset.displayName}`);
67+
console.log(`Dataset example count: ${dataset.exampleCount}`);
68+
console.log(`Text classification type:`);
69+
console.log(
70+
`\t ${dataset.textClassificationDatasetMetadata.classificationType}`
71+
);
72+
console.log(`Dataset create time:`);
73+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
74+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
75+
})
76+
.catch(err => {
77+
console.error(err);
78+
});
79+
// [END automl_natural_language_createDataset]
80+
}
81+
82+
function listDatasets(projectId, computeRegion, filter) {
83+
// [START automl_natural_language_listDatasets]
84+
const automl = require(`@google-cloud/automl`);
85+
86+
const client = new automl.v1beta1.AutoMlClient();
87+
88+
/**
89+
* TODO(developer): Uncomment the following line before running the sample.
90+
*/
91+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
92+
// const computeRegion = `region-name, e.g. "us-central1"`;
93+
// const filter_ = `filter expressions, must specify field e.g. “imageClassificationModelMetadata:*”`;
94+
95+
// A resource that represents a Google Cloud Platform location.
96+
const projectLocation = client.locationPath(projectId, computeRegion);
97+
98+
// List all the datasets available in the region by applying filter.
99+
client
100+
.listDatasets({parent: projectLocation, filter: filter})
101+
.then(responses => {
102+
const datasets = responses[0];
103+
104+
// Display the dataset information.
105+
console.log(`List of datasets:`);
106+
datasets.forEach(dataset => {
107+
console.log(`Dataset name: ${dataset.name}`);
108+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
109+
console.log(`Dataset display name: ${dataset.displayName}`);
110+
console.log(`Dataset example count: ${dataset.exampleCount}`);
111+
console.log(`Text classification type:`);
112+
console.log(
113+
`\t ${dataset.textClassificationDatasetMetadata.classificationType}`
114+
);
115+
console.log(`Dataset create time: `);
116+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
117+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
118+
console.log(`\n`);
119+
});
120+
})
121+
.catch(err => {
122+
console.error(err);
123+
});
124+
// [END automl_natural_language_listDatasets]
125+
}
126+
127+
function getDataset(projectId, computeRegion, datasetId) {
128+
// [START automl_natural_language_getDataset]
129+
const automl = require(`@google-cloud/automl`);
130+
131+
const client = new automl.v1beta1.AutoMlClient();
132+
133+
/**
134+
* TODO(developer): Uncomment the following line before running the sample.
135+
*/
136+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
137+
// const computeRegion = `region-name, e.g. "us-central1"`;
138+
// const datasetId = `Id of the dataset`;
139+
140+
// Get the full path of the dataset.
141+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
142+
143+
// Get complete detail of the dataset.
144+
client
145+
.getDataset({name: datasetFullId})
146+
.then(responses => {
147+
const dataset = responses[0];
148+
149+
// Display the dataset information.
150+
console.log(`Dataset name: ${dataset.name}`);
151+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
152+
console.log(`Dataset display name: ${dataset.displayName}`);
153+
console.log(`Dataset example count: ${dataset.exampleCount}`);
154+
console.log(
155+
`Text classification type: ${
156+
dataset.textClassificationDatasetMetadata.classificationType
157+
}`
158+
);
159+
console.log(`Dataset create time: `);
160+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
161+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
162+
})
163+
.catch(err => {
164+
console.error(err);
165+
});
166+
// [END automl_natural_language_getDataset]
167+
}
168+
169+
function importData(projectId, computeRegion, datasetId, path) {
170+
// [START automl_natural_language_importDataset]
171+
const automl = require(`@google-cloud/automl`);
172+
173+
const client = new automl.v1beta1.AutoMlClient();
174+
175+
/**
176+
* TODO(developer): Uncomment the following line before running the sample.
177+
*/
178+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
179+
// const computeRegion = `region-name, e.g. "us-central1"`;
180+
// const datasetId = `Id of the dataset`;
181+
// const path = `string or array of .csv paths in AutoML Vision CSV format, e.g. “gs://myproject/mytraindata.csv”;`
182+
183+
// Get the full path of the dataset.
184+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
185+
186+
// Get the multiple Google Cloud Storage URIs.
187+
const inputUris = path.split(`,`);
188+
const inputConfig = {
189+
gcsSource: {
190+
inputUris: inputUris,
191+
},
192+
};
193+
194+
// Import the dataset from the input URI.
195+
client
196+
.importData({name: datasetFullId, inputConfig: inputConfig})
197+
.then(responses => {
198+
const operation = responses[0];
199+
console.log(`Processing import...`);
200+
return operation.promise();
201+
})
202+
.then(responses => {
203+
// The final result of the operation.
204+
if (responses[2].done === true) console.log(`Data imported.`);
205+
})
206+
.catch(err => {
207+
console.error(err);
208+
});
209+
// [END automl_natural_language_importDataset]
210+
}
211+
212+
function exportData(projectId, computeRegion, datasetId, outputUri) {
213+
// [START automl_natural_language_exportDataset]
214+
const automl = require(`@google-cloud/automl`);
215+
216+
const client = new automl.v1beta1.AutoMlClient();
217+
218+
/**
219+
* TODO(developer): Uncomment the following line before running the sample.
220+
*/
221+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
222+
// const computeRegion = `region-name, e.g. "us-central1"`;
223+
// const datasetId = `Id of the dataset`;
224+
// const outputUri = `Google Cloud Storage URI for the export directory, e.g. “gs://myproject/output”;`
225+
226+
// Get the full path of the dataset.
227+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
228+
229+
// Set the output URI
230+
const outputConfig = {
231+
gcsDestination: {
232+
outputUriPrefix: outputUri,
233+
},
234+
};
235+
236+
// Export the data to the output URI.
237+
client
238+
.exportData({name: datasetFullId, outputConfig: outputConfig})
239+
.then(responses => {
240+
const operation = responses[0];
241+
console.log(`Processing export...`);
242+
return operation.promise();
243+
})
244+
.then(responses => {
245+
// The final result of the operation.
246+
if (responses[2].done === true) console.log(`Data exported.`);
247+
})
248+
.catch(err => {
249+
console.error(err);
250+
});
251+
// [END automl_natural_language_exportDataset]
252+
}
253+
254+
function deleteDataset(projectId, computeRegion, datasetId) {
255+
// [START automl_natural_language_deleteDataset]
256+
const automl = require(`@google-cloud/automl`);
257+
258+
const client = new automl.v1beta1.AutoMlClient();
259+
260+
/**
261+
* TODO(developer): Uncomment the following line before running the sample.
262+
*/
263+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
264+
// const computeRegion = `region-name, e.g. "us-central1"`;
265+
// const datasetId = `Id of the dataset`;
266+
267+
// Get the full path of the dataset.
268+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
269+
270+
// Delete a dataset.
271+
client
272+
.deleteDataset({name: datasetFullId})
273+
.then(responses => {
274+
const operation = responses[0];
275+
return operation.promise();
276+
})
277+
.then(responses => {
278+
// The final result of the operation.
279+
if (responses[2].done === true) console.log(`Dataset deleted.`);
280+
})
281+
.catch(err => {
282+
console.error(err);
283+
});
284+
// [END automl_natural_language_deleteDataset]
285+
}
286+
287+
require(`yargs`)
288+
.demand(1)
289+
.options({
290+
computeRegion: {
291+
alias: `c`,
292+
type: `string`,
293+
default: process.env.REGION_NAME,
294+
requiresArg: true,
295+
description: `region name e.g. "us-central1"`,
296+
},
297+
datasetName: {
298+
alias: `n`,
299+
type: `string`,
300+
default: `testDataSet`,
301+
requiresArg: true,
302+
description: `Name of the Dataset`,
303+
},
304+
datasetId: {
305+
alias: `i`,
306+
type: `string`,
307+
requiresArg: true,
308+
description: `Id of the dataset`,
309+
},
310+
filter: {
311+
alias: `f`,
312+
default: `text_classification_dataset_metadata:*`,
313+
type: `string`,
314+
requiresArg: false,
315+
description: `filter expression`,
316+
},
317+
multilabel: {
318+
alias: `m`,
319+
type: `string`,
320+
default: false,
321+
requiresArg: true,
322+
description:
323+
`Type of the classification problem, ` +
324+
`False - MULTICLASS, True - MULTILABEL.`,
325+
},
326+
outputUri: {
327+
alias: `o`,
328+
type: `string`,
329+
requiresArg: true,
330+
description: `URI (or local path) to export dataset`,
331+
},
332+
path: {
333+
alias: `p`,
334+
type: `string`,
335+
global: true,
336+
default: `gs://nodejs-docs-samples-vcm/flowerTraindataMini.csv`,
337+
requiresArg: true,
338+
description: `URI or local path to input .csv, or array of .csv paths`,
339+
},
340+
projectId: {
341+
alias: `z`,
342+
type: `number`,
343+
default: process.env.GCLOUD_PROJECT,
344+
requiresArg: true,
345+
description: `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`,
346+
},
347+
})
348+
.command(`create-dataset`, `creates a new Dataset`, {}, opts =>
349+
createDataset(
350+
opts.projectId,
351+
opts.computeRegion,
352+
opts.datasetName,
353+
opts.multilabel
354+
)
355+
)
356+
.command(`list-datasets`, `list all Datasets`, {}, opts =>
357+
listDatasets(opts.projectId, opts.computeRegion, opts.filter)
358+
)
359+
.command(`get-dataset`, `Get a Dataset`, {}, opts =>
360+
getDataset(opts.projectId, opts.computeRegion, opts.datasetId)
361+
)
362+
.command(`delete-dataset`, `Delete a dataset`, {}, opts =>
363+
deleteDataset(opts.projectId, opts.computeRegion, opts.datasetId)
364+
)
365+
.command(`import-data`, `Import labeled items into dataset`, {}, opts =>
366+
importData(opts.projectId, opts.computeRegion, opts.datasetId, opts.path)
367+
)
368+
.command(
369+
`export-data`,
370+
`Export a dataset to a Google Cloud Storage Bucket`,
371+
{},
372+
opts =>
373+
exportData(
374+
opts.projectId,
375+
opts.computeRegion,
376+
opts.datasetId,
377+
opts.outputUri
378+
)
379+
)
380+
.example(`node $0 create-dataset -n "newDataSet"`)
381+
.example(`node $0 list-datasets -f "imageClassificationDatasetMetadata:*"`)
382+
.example(`node $0 get-dataset -i "DATASETID"`)
383+
.example(`node $0 delete-dataset -i "DATASETID"`)
384+
.example(
385+
`node $0 import-data -i "dataSetId" -p "gs://myproject/mytraindata.csv"`
386+
)
387+
.example(
388+
`node $0 export-data -i "dataSetId" -o "gs://myproject/outputdestination.csv"`
389+
)
390+
.wrap(120)
391+
.recommendCommands()
392+
.help()
393+
.strict().argv;

0 commit comments

Comments
 (0)