1818import java .util .Optional ;
1919import java .util .Queue ;
2020import java .util .Set ;
21+ import lombok .Builder ;
2122import lombok .Value ;
22- import org .slf4j .Logger ;
23- import org .slf4j .LoggerFactory ;
2423
24+ @ Builder
2525public class JsonSecretsProcessor {
2626
27- private static final Logger LOGGER = LoggerFactory .getLogger (JsonSecretsProcessor .class );
27+ @ Builder .Default
28+ private boolean maskSecrets = true ;
2829
29- public static final String AIRBYTE_SECRET_FIELD = "airbyte_secret" ;
30- public static final String PROPERTIES_FIELD = "properties" ;
31- public static final String TYPE_FIELD = "type" ;
32- public static final String ARRAY_TYPE_FIELD = "array" ;
33- public static final String ITEMS_FIELD = "items" ;
30+ @ Builder .Default
31+ private boolean copySecrets = true ;
3432
35- private static final JsonSchemaValidator VALIDATOR = new JsonSchemaValidator ();
33+ protected static final JsonSchemaValidator VALIDATOR = new JsonSchemaValidator ();
3634
3735 @ VisibleForTesting
3836 static String SECRETS_MASK = "**********" ;
3937
38+ static final String AIRBYTE_SECRET_FIELD = "airbyte_secret" ;
39+ static final String PROPERTIES_FIELD = "properties" ;
40+ static final String TYPE_FIELD = "type" ;
41+ static final String ARRAY_TYPE_FIELD = "array" ;
42+ static final String ITEMS_FIELD = "items" ;
43+
4044 /**
4145 * Returns a copy of the input object wherein any fields annotated with "airbyte_secret" in the
4246 * input schema are masked.
@@ -47,20 +51,97 @@ public class JsonSecretsProcessor {
4751 * @param schema Schema containing secret annotations
4852 * @param obj Object containing potentially secret fields
4953 */
50- // todo: fix bug where this doesn't handle non-oneof nesting or just arrays
51- // see: https://github.com/airbytehq/airbyte/issues/6393
52- public JsonNode maskSecrets (final JsonNode obj , final JsonNode schema ) {
53- // if schema is an object and has a properties field
54- if (!canBeProcessed (schema )) {
55- return obj ;
54+ public JsonNode prepareSecretsForOutput (final JsonNode obj , final JsonNode schema ) {
55+ if (maskSecrets ) {
56+ // if schema is an object and has a properties field
57+ if (!canBeProcessed (schema )) {
58+ return obj ;
59+ }
60+
61+ final SecretKeys secretKeys = getAllSecretKeys (schema );
62+ return maskAllSecrets (obj , secretKeys );
63+ }
64+
65+ return obj ;
66+ }
67+
68+ /**
69+ * Returns a copy of the destination object in which any secret fields (as denoted by the input
70+ * schema) found in the source object are added.
71+ * <p>
72+ * This method absorbs secrets both at the top level of the configuration object and in nested
73+ * properties in a oneOf.
74+ *
75+ * @param src The object potentially containing secrets
76+ * @param dst The object to absorb secrets into
77+ * @param schema
78+ * @return
79+ */
80+ public JsonNode copySecrets (final JsonNode src , final JsonNode dst , final JsonNode schema ) {
81+ if (copySecrets ) {
82+ if (!canBeProcessed (schema )) {
83+ return dst ;
84+ }
85+ Preconditions .checkArgument (dst .isObject ());
86+ Preconditions .checkArgument (src .isObject ());
87+
88+ final ObjectNode dstCopy = dst .deepCopy ();
89+
90+ final ObjectNode properties = (ObjectNode ) schema .get (PROPERTIES_FIELD );
91+ for (final String key : Jsons .keys (properties )) {
92+ // If the source object doesn't have this key then we have nothing to copy, so we should skip to the
93+ // next key.
94+ if (!src .has (key )) {
95+ continue ;
96+ }
97+
98+ final JsonNode fieldSchema = properties .get (key );
99+ // We only copy the original secret if the destination object isn't attempting to overwrite it
100+ // I.e. if the destination object's value is set to the mask, then we can copy the original secret
101+ if (JsonSecretsProcessor .isSecret (fieldSchema ) && dst .has (key ) && dst .get (key ).asText ().equals (SECRETS_MASK )) {
102+ dstCopy .set (key , src .get (key ));
103+ } else if (dstCopy .has (key )) {
104+ // If the destination has this key, then we should consider copying it
105+
106+ // Check if this schema is a combination node; if it is, find a matching sub-schema and copy based
107+ // on that sub-schema
108+ final var combinationKey = findJsonCombinationNode (fieldSchema );
109+ if (combinationKey .isPresent ()) {
110+ var combinationCopy = dstCopy .get (key );
111+ final var arrayNode = (ArrayNode ) fieldSchema .get (combinationKey .get ());
112+ for (int i = 0 ; i < arrayNode .size (); i ++) {
113+ final JsonNode childSchema = arrayNode .get (i );
114+ /*
115+ * when traversing a oneOf or anyOf if multiple schema in the oneOf or anyOf have the SAME key, but
116+ * a different type, then, without this test, we can try to apply the wrong schema to the object
117+ * resulting in errors because of type mismatches.
118+ */
119+ if (VALIDATOR .test (childSchema , combinationCopy )) {
120+ // Absorb field values if any of the combination option is declaring it as secrets
121+ combinationCopy = copySecrets (src .get (key ), combinationCopy , childSchema );
122+ }
123+ }
124+ dstCopy .set (key , combinationCopy );
125+ } else {
126+ // Otherwise, this is just a plain old json node; recurse into it. If it's not actually an object,
127+ // the recursive call will exit immediately.
128+ final JsonNode copiedField = copySecrets (src .get (key ), dstCopy .get (key ), fieldSchema );
129+ dstCopy .set (key , copiedField );
130+ }
131+ }
132+ }
133+
134+ return dstCopy ;
56135 }
57- Preconditions .checkArgument (schema .isObject ());
58136
59- final SecretKeys secretKeys = getAllSecretKeys (schema );
60- return maskAllSecrets (obj , secretKeys );
137+ return src ;
61138 }
62139
63- private JsonNode maskAllSecrets (final JsonNode obj , final SecretKeys secretKeys ) {
140+ static boolean isSecret (final JsonNode obj ) {
141+ return obj .isObject () && obj .has (AIRBYTE_SECRET_FIELD ) && obj .get (AIRBYTE_SECRET_FIELD ).asBoolean ();
142+ }
143+
144+ protected JsonNode maskAllSecrets (final JsonNode obj , final SecretKeys secretKeys ) {
64145 final JsonNode copiedObj = obj .deepCopy ();
65146 final Queue <JsonNode > toProcess = new LinkedList <>();
66147 toProcess .add (copiedObj );
@@ -91,14 +172,14 @@ private JsonNode maskAllSecrets(final JsonNode obj, final SecretKeys secretKeys)
91172 }
92173
93174 @ Value
94- private class SecretKeys {
175+ protected class SecretKeys {
95176
96177 private final Set <String > fieldSecretKey ;
97178 private final Set <String > arraySecretKey ;
98179
99180 }
100181
101- private SecretKeys getAllSecretKeys (final JsonNode schema ) {
182+ protected SecretKeys getAllSecretKeys (final JsonNode schema ) {
102183 final Set <String > fieldSecretKeys = new HashSet <>();
103184 final Set <String > arraySecretKeys = new HashSet <>();
104185
@@ -115,7 +196,7 @@ private SecretKeys getAllSecretKeys(final JsonNode schema) {
115196 } else {
116197 toProcess .add (arrayItems );
117198 }
118- } else if (isSecret (currentNode .get (key ))) {
199+ } else if (JsonSecretsProcessor . isSecret (currentNode .get (key ))) {
119200 fieldSecretKeys .add (key );
120201 } else if (currentNode .get (key ).isObject ()) {
121202 toProcess .add (currentNode .get (key ));
@@ -140,77 +221,6 @@ public static Optional<String> findJsonCombinationNode(final JsonNode node) {
140221 return Optional .empty ();
141222 }
142223
143- /**
144- * Returns a copy of the destination object in which any secret fields (as denoted by the input
145- * schema) found in the source object are added.
146- * <p>
147- * This method absorbs secrets both at the top level of the configuration object and in nested
148- * properties in a oneOf.
149- *
150- * @param src The object potentially containing secrets
151- * @param dst The object to absorb secrets into
152- * @param schema
153- * @return
154- */
155- public JsonNode copySecrets (final JsonNode src , final JsonNode dst , final JsonNode schema ) {
156- if (!canBeProcessed (schema )) {
157- return dst ;
158- }
159- Preconditions .checkArgument (dst .isObject ());
160- Preconditions .checkArgument (src .isObject ());
161-
162- final ObjectNode dstCopy = dst .deepCopy ();
163-
164- final ObjectNode properties = (ObjectNode ) schema .get (PROPERTIES_FIELD );
165- for (final String key : Jsons .keys (properties )) {
166- // If the source object doesn't have this key then we have nothing to copy, so we should skip to the
167- // next key.
168- if (!src .has (key )) {
169- continue ;
170- }
171-
172- final JsonNode fieldSchema = properties .get (key );
173- // We only copy the original secret if the destination object isn't attempting to overwrite it
174- // i.e: if the value of the secret isn't set to the mask
175- if (isSecret (fieldSchema ) && dst .has (key ) && dst .get (key ).asText ().equals (SECRETS_MASK )) {
176- dstCopy .set (key , src .get (key ));
177- } else if (dstCopy .has (key )) {
178- // If the destination has this key, then we should consider copying it
179-
180- // Check if this schema is a combination node; if it is, find a matching sub-schema and copy based
181- // on that sub-schema
182- final var combinationKey = findJsonCombinationNode (fieldSchema );
183- if (combinationKey .isPresent ()) {
184- var combinationCopy = dstCopy .get (key );
185- final var arrayNode = (ArrayNode ) fieldSchema .get (combinationKey .get ());
186- for (int i = 0 ; i < arrayNode .size (); i ++) {
187- final JsonNode childSchema = arrayNode .get (i );
188- /*
189- * when traversing a oneOf or anyOf if multiple schema in the oneOf or anyOf have the SAME key, but
190- * a different type, then, without this test, we can try to apply the wrong schema to the object
191- * resulting in errors because of type mismatches.
192- */
193- if (VALIDATOR .test (childSchema , combinationCopy )) {
194- // Absorb field values if any of the combination option is declaring it as secrets
195- combinationCopy = copySecrets (src .get (key ), combinationCopy , childSchema );
196- }
197- }
198- dstCopy .set (key , combinationCopy );
199- } else {
200- // Otherwise, this is just a plain old json object; recurse into it.
201- final JsonNode copiedField = copySecrets (src .get (key ), dstCopy .get (key ), fieldSchema );
202- dstCopy .set (key , copiedField );
203- }
204- }
205- }
206-
207- return dstCopy ;
208- }
209-
210- public static boolean isSecret (final JsonNode obj ) {
211- return obj .isObject () && obj .has (AIRBYTE_SECRET_FIELD ) && obj .get (AIRBYTE_SECRET_FIELD ).asBoolean ();
212- }
213-
214224 public static boolean canBeProcessed (final JsonNode schema ) {
215225 return schema .isObject () && schema .has (PROPERTIES_FIELD ) && schema .get (PROPERTIES_FIELD ).isObject ();
216226 }
0 commit comments