Skip to content

Commit d8e2680

Browse files
bzhangambzhangam
authored
Add example plugins for mapping transformer and system ingest processor. (#18300)
Signed-off-by: Bo Zhang <[email protected]>
1 parent e9c41f7 commit d8e2680

File tree

20 files changed

+952
-0
lines changed

20 files changed

+952
-0
lines changed

‎plugins/examples/mapping-transformer/build.gradle

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
apply plugin: 'opensearch.opensearchplugin'
12+
apply plugin: 'opensearch.yaml-rest-test'
13+
14+
opensearchplugin {
15+
name = 'example-mapping-transformer'
16+
description = 'An example plugin implementing a mapping transformer.'
17+
classname = 'org.opensearch.example.mappingtransformer.ExampleMappingTransformerPlugin'
18+
licenseFile = rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
19+
noticeFile = rootProject.file('NOTICE.txt')
20+
}

‎plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformer.java

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.example.mappingtransformer;
10+
11+
import org.opensearch.core.action.ActionListener;
12+
import org.opensearch.index.mapper.MappingTransformer;
13+
14+
import java.util.HashMap;
15+
import java.util.Map;
16+
17+
/**
18+
* This example mapping transformer will automatically add a text field to the mapping if it finds a
19+
* mapping_transform_trigger_field.
20+
*/
21+
public class ExampleMappingTransformer implements MappingTransformer {
22+
/**
23+
* Constructs a new ExampleMappingTransformer
24+
*/
25+
public ExampleMappingTransformer() {}
26+
27+
/**
28+
* Name for doc. Actions like create index and legacy create/update index template will have the
29+
* mapping properties under a _doc key.
30+
*/
31+
public static final String DOC = "_doc";
32+
/**
33+
* Name for properties. An object field will define subfields as properties.
34+
*/
35+
public static final String PROPERTIES = "properties";
36+
37+
/**
38+
* The name of the trigger field. The trigger field will trigger the logic to transform the mapping.
39+
*/
40+
public static final String TRIGGER_FIELD_NAME = "mapping_transform_trigger_field";
41+
/**
42+
* The name of the auto added field.
43+
*/
44+
public static final String AUTO_ADDED_FIELD_NAME = "field_auto_added_by_example_mapping_transformer";
45+
46+
@Override
47+
public void transform(Map<String, Object> mapping, TransformContext context, ActionListener<Void> listener) {
48+
Map<String, Object> properties = getProperties(mapping);
49+
if (properties.containsKey(TRIGGER_FIELD_NAME)) {
50+
properties.put(AUTO_ADDED_FIELD_NAME, Map.of("type", "text"));
51+
}
52+
listener.onResponse(null);
53+
}
54+
55+
/**
56+
* Help extract the properties from a mapping
57+
* @param mapping index mapping
58+
* @return properties of the mapping
59+
*/
60+
@SuppressWarnings("unchecked")
61+
private Map<String, Object> getProperties(Map<String, Object> mapping) {
62+
if (mapping == null) {
63+
return new HashMap<>();
64+
}
65+
66+
if (mapping.containsKey(DOC) && mapping.get(DOC) instanceof Map) {
67+
Map<String, Object> doc = (Map<String, Object>) mapping.get(DOC);
68+
if (doc.containsKey(PROPERTIES) && doc.get(PROPERTIES) instanceof Map) {
69+
return (Map<String, Object>) doc.get(PROPERTIES);
70+
}
71+
} else if (mapping.containsKey(PROPERTIES) && mapping.get(PROPERTIES) instanceof Map) {
72+
return (Map<String, Object>) mapping.get(PROPERTIES);
73+
}
74+
75+
return new HashMap<>();
76+
}
77+
}

‎plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerPlugin.java

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.example.mappingtransformer;
10+
11+
import org.opensearch.index.mapper.MappingTransformer;
12+
import org.opensearch.plugins.MapperPlugin;
13+
import org.opensearch.plugins.Plugin;
14+
15+
import java.util.List;
16+
17+
/**
18+
* Example plugin that implements a custom mapping transformer.
19+
*/
20+
public class ExampleMappingTransformerPlugin extends Plugin implements MapperPlugin {
21+
22+
/**
23+
* Constructs a new ExampleMappingTransformerPlugin
24+
*/
25+
public ExampleMappingTransformerPlugin() {}
26+
27+
/**
28+
* @return Available mapping transformer
29+
*/
30+
@Override
31+
public List<MappingTransformer> getMappingTransformers() {
32+
return List.of(new ExampleMappingTransformer());
33+
}
34+
}

‎plugins/examples/mapping-transformer/src/main/java/org/opensearch/example/mappingtransformer/package-info.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
/**
10+
* Example classes demonstrating the use of a custom mapping transformer in a plugin.
11+
*/
12+
package org.opensearch.example.mappingtransformer;

‎plugins/examples/mapping-transformer/src/test/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerPluginTests.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.example.mappingtransformer;
10+
11+
import org.opensearch.index.mapper.MappingTransformer;
12+
import org.opensearch.test.OpenSearchTestCase;
13+
14+
import java.util.List;
15+
16+
public class ExampleMappingTransformerPluginTests extends OpenSearchTestCase {
17+
private final ExampleMappingTransformerPlugin plugin = new ExampleMappingTransformerPlugin();
18+
19+
public void testGetMappingTransformers() {
20+
List<MappingTransformer> mappingTransformerList = plugin.getMappingTransformers();
21+
22+
assertTrue("Should return an example mapping transformer.", mappingTransformerList.getFirst() instanceof ExampleMappingTransformer);
23+
}
24+
}

‎plugins/examples/mapping-transformer/src/test/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerTests.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.example.mappingtransformer;
10+
11+
import org.opensearch.action.support.PlainActionFuture;
12+
import org.opensearch.test.OpenSearchTestCase;
13+
14+
import java.util.HashMap;
15+
import java.util.Map;
16+
17+
import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.AUTO_ADDED_FIELD_NAME;
18+
import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.DOC;
19+
import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.PROPERTIES;
20+
import static org.opensearch.example.mappingtransformer.ExampleMappingTransformer.TRIGGER_FIELD_NAME;
21+
22+
public class ExampleMappingTransformerTests extends OpenSearchTestCase {
23+
private final ExampleMappingTransformer transformer = new ExampleMappingTransformer();
24+
25+
public void testExampleMappingTransformer_whenMappingWithoutDocLayer() {
26+
Map<String, Object> mapping = new HashMap<>();
27+
Map<String, Object> properties = new HashMap<>();
28+
mapping.put(PROPERTIES, properties);
29+
properties.put("dummyField", Map.of("type", "text"));
30+
31+
transformer.transform(mapping, null, new PlainActionFuture<>());
32+
33+
// verify no field is auto-injected
34+
assertFalse(
35+
"No trigger field in the mapping so should not transform the mapping to inject the field.",
36+
properties.containsKey(AUTO_ADDED_FIELD_NAME)
37+
);
38+
39+
properties.put(TRIGGER_FIELD_NAME, Map.of("type", "text"));
40+
transformer.transform(mapping, null, new PlainActionFuture<>());
41+
42+
// verify the field should be auto added to the mapping
43+
assertTrue(
44+
"The mapping has the trigger field so should transform the mapping to auto inject a field.",
45+
properties.containsKey(AUTO_ADDED_FIELD_NAME)
46+
);
47+
}
48+
49+
public void testExampleMappingTransformer_whenMappingWithDocLayer() {
50+
Map<String, Object> doc = new HashMap<>();
51+
Map<String, Object> mapping = new HashMap<>();
52+
Map<String, Object> properties = new HashMap<>();
53+
doc.put(DOC, mapping);
54+
mapping.put(PROPERTIES, properties);
55+
properties.put("dummyField", Map.of("type", "text"));
56+
57+
transformer.transform(doc, null, new PlainActionFuture<>());
58+
59+
// verify no field is auto-injected
60+
assertFalse(
61+
"No trigger field in the mapping so should not transform the mapping to inject the field.",
62+
properties.containsKey(AUTO_ADDED_FIELD_NAME)
63+
);
64+
65+
properties.put(TRIGGER_FIELD_NAME, Map.of("type", "text"));
66+
transformer.transform(mapping, null, new PlainActionFuture<>());
67+
68+
// verify the field should be auto added to the mapping
69+
assertTrue(
70+
"The mapping has the trigger field so should transform the mapping to auto inject a field.",
71+
properties.containsKey(AUTO_ADDED_FIELD_NAME)
72+
);
73+
}
74+
75+
public void testExampleMappingTransformer_whenNoMapping_thenDoNothing() {
76+
transformer.transform(null, null, new PlainActionFuture<>());
77+
}
78+
79+
public void testExampleMappingTransformer_whenEmptyMapping_thenDoNothing() {
80+
Map<String, Object> doc = new HashMap<>();
81+
transformer.transform(doc, null, new PlainActionFuture<>());
82+
assertEquals(new HashMap<>(), doc);
83+
}
84+
}

‎plugins/examples/mapping-transformer/src/yamlRestTest/java/org/opensearch/example/mappingtransformer/ExampleMappingTransformerClientYamlTestSuiteIT.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.example.mappingtransformer;
10+
11+
import com.carrotsearch.randomizedtesting.annotations.Name;
12+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
13+
14+
import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
15+
import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
16+
17+
public class ExampleMappingTransformerClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
18+
19+
public ExampleMappingTransformerClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
20+
super(testCandidate);
21+
}
22+
23+
@ParametersFactory
24+
public static Iterable<Object[]> parameters() throws Exception {
25+
return OpenSearchClientYamlSuiteTestCase.createParameters();
26+
}
27+
}

‎plugins/examples/mapping-transformer/src/yamlRestTest/resources/rest-api-spec/test/example-mapping-transformer/10_basic.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Integration tests for the mapping transformer example plugin
2+
#
3+
"Plugin loaded":
4+
- skip:
5+
reason: "contains is a newly added assertion"
6+
features: contains
7+
- do:
8+
cluster.state: {}
9+
10+
# Get cluster-manager node id
11+
- set: { cluster_manager_node: cluster_manager }
12+
13+
- do:
14+
nodes.info: {}
15+
16+
- contains: { nodes.$cluster_manager.plugins: { name: example-mapping-transformer } }

‎plugins/examples/mapping-transformer/src/yamlRestTest/resources/rest-api-spec/test/example-mapping-transformer/20_mapping_transformer.yml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
---
2+
"Test auto-injected field on index creation":
3+
- do:
4+
indices.create:
5+
index: test-index
6+
body:
7+
mappings:
8+
properties:
9+
mapping_transform_trigger_field:
10+
type: keyword
11+
- do:
12+
indices.get_mapping:
13+
index: test-index
14+
- match:
15+
test-index.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
16+
17+
---
18+
"Test auto-injected field on mapping update":
19+
- do:
20+
indices.create:
21+
index: test-index-update
22+
body:
23+
mappings:
24+
properties:
25+
dummy_field:
26+
type: keyword
27+
- do:
28+
indices.put_mapping:
29+
index: test-index-update
30+
body:
31+
properties:
32+
mapping_transform_trigger_field:
33+
type: keyword
34+
- do:
35+
indices.get_mapping:
36+
index: test-index-update
37+
- match:
38+
test-index-update.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
39+
40+
---
41+
"Test auto-injected field via index template":
42+
- do:
43+
indices.put_index_template:
44+
name: example_template
45+
body:
46+
index_patterns: ["auto-template-*"]
47+
template:
48+
mappings:
49+
properties:
50+
mapping_transform_trigger_field:
51+
type: keyword
52+
- do:
53+
indices.create:
54+
index: auto-template-1
55+
- do:
56+
indices.get_mapping:
57+
index: auto-template-1
58+
- match:
59+
auto-template-1.mappings.properties.mapping_transform_trigger_field.type: keyword
60+
- match:
61+
auto-template-1.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text
62+
63+
---
64+
"Test auto-injected field via legacy create template API":
65+
- do:
66+
indices.put_template:
67+
name: legacy_template
68+
body:
69+
index_patterns: ["legacy-*"]
70+
mappings:
71+
properties:
72+
mapping_transform_trigger_field:
73+
type: keyword
74+
- do:
75+
indices.create:
76+
index: legacy-1
77+
- do:
78+
indices.get_mapping:
79+
index: legacy-1
80+
- match:
81+
legacy-1.mappings.properties.mapping_transform_trigger_field.type: keyword
82+
- match:
83+
legacy-1.mappings.properties.field_auto_added_by_example_mapping_transformer.type: text

0 commit comments

Comments
 (0)