feat: Add configurable prop types to neo4j csv publisher (#1993)

* Add configurable prop types to neo4j csv publisher

Signed-off-by: Kristen Armes <[email protected]>

* Bump databuilder version

Signed-off-by: Kristen Armes <[email protected]>

Signed-off-by: Kristen Armes <[email protected]>

Author: kristenarmes

databuilder/databuilder/publisher/neo4j_csv_unwind_publisher.py

@@ -91,6 +91,8 @@ def init(self, conf: ConfigTree) -> None:
         self._publish_reverse_relationships: bool = conf.get_bool(PublishBehaviorConfigs.PUBLISH_REVERSE_RELATIONSHIPS)
         self._preserve_adhoc_ui_data = conf.get_bool(PublishBehaviorConfigs.PRESERVE_ADHOC_UI_DATA)
         self._preserve_empty_props: bool = conf.get_bool(PublishBehaviorConfigs.PRESERVE_EMPTY_PROPS)
+        self._prop_types_to_configure: Dict =\
+            dict(conf.get(Neo4jCsvPublisherConfigs.NEO4J_PROP_TYPES_TO_CONFIGURE, default={}))
         if self._add_publisher_metadata and not self._publish_tag:
             raise Exception(f'{PublisherConfigs.JOB_PUBLISH_TAG} should not be empty')
 
@@ -317,23 +319,38 @@ def _create_props_body(self,
         template = Template("""
             {% for k in record_keys %}
                 {% if preserve_empty_props %}
-                    {{ identifier }}.{{ k }} = row.{{ k }}
+                    {% if k in prop_types_to_configure %}
+                        {{ identifier }}.{{ k }} = {{ prop_types_to_configure[k] }}(row.{{ k }})
+                    {% else %}
+                        {{ identifier }}.{{ k }} = row.{{ k }}
+                    {% endif %}
                 {% else %}
-                    {{ identifier }}.{{ k }} = (CASE row.{{ k }} WHEN '' THEN NULL ELSE row.{{ k }} END)
+                    {% if k in prop_types_to_configure %}
+                        {{ identifier }}.{{ k }} =
+                        (CASE row.{{ k }} WHEN '' THEN NULL ELSE {{ prop_types_to_configure[k] }}(row.{{ k }}) END)
+                    {% else %}
+                        {{ identifier }}.{{ k }} = (CASE row.{{ k }} WHEN '' THEN NULL ELSE row.{{ k }} END)
+                    {% endif %}
                 {% endif %}
                 {{ ", " if not loop.last else "" }}
             {% endfor %}
             {% if record_keys and add_publisher_metadata %}
                 ,
             {% endif %}
             {% if add_publisher_metadata %}
-                {{ identifier }}.{{ published_tag_prop }} = '{{ publish_tag }}',
+                {% if published_tag_prop in prop_types_to_configure %}
+                    {{ identifier }}.{{ published_tag_prop }} =
+                    {{ prop_types_to_configure[published_tag_prop] }}('{{ publish_tag }}'),
+                {% else %}
+                    {{ identifier }}.{{ published_tag_prop }} = '{{ publish_tag }}',
+                {% endif %}
                 {{ identifier }}.{{ last_updated_prop }} = timestamp()
             {% endif %}
         """)
 
         props_body = template.render(record_keys=record_keys,
                                      preserve_empty_props=self._preserve_empty_props,
+                                     prop_types_to_configure=self._prop_types_to_configure,
                                      identifier=identifier,
                                      add_publisher_metadata=self._add_publisher_metadata,
                                      published_tag_prop=PublisherConfigs.PUBLISHED_TAG_PROPERTY_NAME,

databuilder/databuilder/publisher/publisher_config_constants.py

@@ -64,3 +64,10 @@ class Neo4jCsvPublisherConfigs:
     # NEO4J_VALIDATE_SSL is a boolean indicating whether to validate the server's SSL/TLS
     # cert against system CAs
     NEO4J_VALIDATE_SSL = 'neo4j_validate_ssl'
+
+    # This should be a dict using property names as keys mapped to the function name used to configure a specific
+    # type for that property. The values of the properties should be in the correct format that the function accepts.
+    # Example: a config of {'start_time': 'datetime', 'publish_tag': 'date'} where the property values are in the
+    # format <date>T<time> and <date> would apply datetime(n.start_time) and date(n.publish_tag) in the prop merge
+    # statement to create the props as DateTime and Date types instead of strings.
+    NEO4J_PROP_TYPES_TO_CONFIGURE = 'neo4j_prop_types_to_configure'

databuilder/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '7.3.0'
+__version__ = '7.4.0'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  'requirements.txt')