Merge pull request #43 from NetTopologySuite/fix/issue031_null_attribute_value

Allow writing shapefiles containing null attributes.

Author: KubaSzostak

README.md

@@ -61,6 +61,8 @@ foreach (var feature in Shapefile.ReadAllFeatures(shpPath))
 
 ### Writing shapefiles using c# code
 
+The most common variant of writing shapefiles is to use `Shapefile.WriteAllFeatures` method.
+
 ```c#
 var features = new List<Feature>();
 for (int i = 1; i < 5; i++)
@@ -89,6 +91,44 @@ for (int i = 1; i < 5; i++)
 Shapefile.WriteAllFeatures(features, shpPath);
 ```
 
+The most efficient way to write large shapefiles is to use `ShapefileWriter` class.
+This variant should also be used when you need to write a shapefile with a attributes containing `null` values.
+
+```c#
+var fields = new List<DbfField>();
+var dateField = fields.AddDateField("date");
+var floatField = fields.AddFloatField("float");
+var intField = fields.AddNumericInt32Field("int");
+var logicalField = fields.AddLogicalField("logical");
+var textField = fields.AddCharacterField("text");
+
+var options = new ShapefileWriterOptions(ShapeType.PolyLine, fields.ToArray());
+using (var shpWriter = Shapefile.OpenWrite(shpPath, options))
+{
+    for (var i = 1; i < 5; i++)
+    {
+        var lineCoords = new List<Coordinate>
+        {
+            new(i, i + 1),
+            new(i, i),
+            new(i + 1, i)
+        };
+        var line = new LineString(lineCoords.ToArray());
+        var mline = new MultiLineString(new LineString[] { line });
+
+        int? nullIntValue = null;
+
+        shpWriter.Geometry = mline;
+        dateField.DateValue = DateTime.Now;
+        floatField.NumericValue = i * 0.1;
+        intField.NumericValue = nullIntValue;
+        logicalField.LogicalValue = i % 2 == 0;
+        textField.StringValue = i.ToString("0.00");
+        shpWriter.Write();
+    }
+}
+```
+
 ## Encoding
 
 The .NET Framework supports a large number of character encodings and code pages.

src/NetTopologySuite.IO.Esri.Shapefile/Helpers/FeatureExtensions.cs

@@ -74,6 +74,78 @@ internal static DbfField[] GetDbfFields(this IAttributesTable attributes)
         }
 
 
+        internal static DbfField[] GetDbfFields(this IEnumerable<IFeature> features)
+        {
+            if (features == null || !features.Any())
+            {
+                return null;
+            }
+
+            var allAttributeNames = new HashSet<string>();
+            var resolvedAttributeNames = new List<string>(); // preserve order of fields
+            var attributeTypes = new Dictionary<string, Type>();
+
+            foreach (var feature in features)
+            {
+                AddAttributes(allAttributeNames, resolvedAttributeNames, attributeTypes, feature.Attributes);
+                if (allAttributeNames.Count == resolvedAttributeNames.Count)
+                {
+                    return GetDbfFields(resolvedAttributeNames, attributeTypes);
+                }
+            }
+
+            var missingAttributeNames = allAttributeNames.Except(resolvedAttributeNames);
+            throw new ShapefileException("Cannot determine DBF attribut type for following attributes: " + string.Join(", ", missingAttributeNames));
+        }
+
+
+        private static void AddAttributes(HashSet<string> allAttributeNames, List<string> resolvedAttributeNames, Dictionary<string, Type> attributeTypes, IAttributesTable attributes)
+        {
+            if (attributes == null)
+            {
+                return;
+            }
+
+            foreach (var attributeName in attributes.GetNames())
+            {
+                if (attributeTypes.ContainsKey(attributeName))
+                {
+                    continue;
+                }
+
+                allAttributeNames.Add(attributeName);
+
+                var attributeType = attributes.GetType(attributeName);
+                if (attributeType != typeof(object))
+                {
+                    resolvedAttributeNames.Add(attributeName); // preserve order of fields
+                    attributeTypes.Add(attributeName, attributeType);
+                }
+            }
+        }
+
+
+        private static DbfField[] GetDbfFields(List<string> attributeNames, Dictionary<string, Type> attributeTypes)
+        {
+            if (attributeNames.Count == 0 || attributeTypes.Count == 0)
+            {
+                return null;
+            }
+
+            var fields = new DbfField[attributeNames.Count];
+
+            for (int i = 0; i < attributeNames.Count; i++)
+            {
+                var name = attributeNames[i];
+                var type = attributeTypes[name];
+                fields[i] = DbfField.Create(name, type);
+            }
+
+            return fields;
+        }
+
+
+
         private static Geometry FindNonEmptyGeometry(Geometry geometry)
         {
             if (geometry == null || geometry.IsEmpty)

src/NetTopologySuite.IO.Esri.Shapefile/Shapefile.cs

@@ -313,9 +313,12 @@ public static void WriteAllFeatures(IEnumerable<IFeature> features, string shpPa
             if (features == null)
                 throw new ArgumentNullException(nameof(features));
 
-            var firstFeature = features.FirstOrDefault()
-                ?? throw new ArgumentException(nameof(ShapefileWriter) + " requires at least one feature to be written.");
-            var fields = firstFeature.Attributes.GetDbfFields();
+            if (!features.Any())
+            {
+                throw new ArgumentException(nameof(ShapefileWriter) + " requires at least one feature to be written.");
+            }
+
+            var fields = features.GetDbfFields();
             var shapeType = features.FindNonEmptyGeometry().GetShapeType();
             var options = new ShapefileWriterOptions(shapeType, fields)
             {

test/NetTopologySuite.IO.Esri.Test/Issues/Issue031.cs

@@ -0,0 +1,181 @@
+using NetTopologySuite.Features;
+using NetTopologySuite.Geometries;
+using NetTopologySuite.IO.Esri.Dbf.Fields;
+using NetTopologySuite.IO.Esri.Shapefiles.Writers;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+
+namespace NetTopologySuite.IO.Esri.Test.Issues;
+
+/// <summary>
+/// https://github.com/NetTopologySuite/NetTopologySuite.IO.Esri/issues/31
+/// </summary>
+internal class Issue031
+{
+
+    [Test]
+    public void CreateShp_NullInt_Auto()
+    {
+        var features = new List<Feature>();
+        for (var i = 1; i < 5; i++)
+        {
+            var lineCoords = new List<Coordinate>
+            {
+                new(i, i + 1),
+                new(i, i),
+                new(i + 1, i)
+            };
+            var line = new LineString(lineCoords.ToArray());
+            var mline = new MultiLineString(new LineString[] { line });
+
+            // When all features have a null value, the field type cannot be detected correctly.
+            int? nullableIntValue = i % 3 == 0 ? 1 : null;
+
+            var attributes = new AttributesTable
+            {
+                { "date", new DateTime() },
+                { "float", i * 0.1 },
+                { "int", nullableIntValue },
+                { "logical", i % 2 == 0 },
+                { "text", i.ToString("0.00") }
+            };
+            
+
+            var feature = new Feature(mline, attributes);
+            features.Add(feature);
+        }
+
+        var shpPath = TestShapefiles.GetTempShpPath();
+        Shapefile.WriteAllFeatures(features, shpPath);
+        TestShapefiles.DeleteShp(shpPath);
+    }
+
+    [Test]
+    public void CreateShp_NullInt_Auto_AllNullError()
+    {
+        var features = new List<Feature>();
+        for (var i = 1; i < 5; i++)
+        {
+            var lineCoords = new List<Coordinate>
+            {
+                new(i, i + 1),
+                new(i, i),
+                new(i + 1, i)
+            };
+            var line = new LineString(lineCoords.ToArray());
+            var mline = new MultiLineString(new LineString[] { line });
+
+            int? nullableIntValue = null;
+
+            var attributes = new AttributesTable
+            {
+                { "date", new DateTime() },
+                { "float", i * 0.1 },
+                { "int", nullableIntValue },
+                { "logical", i % 2 == 0 },
+                { "text", i.ToString("0.00") }
+            };
+
+
+            var feature = new Feature(mline, attributes);
+            features.Add(feature);
+        }
+
+        var shpPath = TestShapefiles.GetTempShpPath();
+
+        // When all features have a null value, the field type cannot be detected correctly.
+        // To solve this, the `AttributesTable` needs to be extended to store attribute types along with attribute values,
+        // so that `AttributesTable.GetType()` returns propert attribute type instead of default `typeof(object)`.
+        // Se also: https://github.com/NetTopologySuite/NetTopologySuite.IO.Esri/issues/31#issuecomment-1975112219
+        var exception = Assert.Throws<ShapefileException>(() => Shapefile.WriteAllFeatures(features, shpPath));
+
+        TestShapefiles.DeleteShp(shpPath);
+        Console.WriteLine(exception.Message);
+    }
+
+    [Test]
+    public void CreateShp_NullInt_Manual_AttributeTable()
+    {
+        var features = new List<Feature>();
+        for (var i = 1; i < 5; i++)
+        {
+            var lineCoords = new List<Coordinate>
+            {
+                new(i, i + 1),
+                new(i, i),
+                new(i + 1, i)
+            };
+            var line = new LineString(lineCoords.ToArray());
+            var mline = new MultiLineString(new LineString[] { line });
+
+            int? nullableIntValue = null;
+
+            var attributes = new AttributesTable
+            {
+                { "date", new DateTime() },
+                { "float", i * 0.1 },
+                { "int", nullableIntValue },
+                { "logical", i % 2 == 0 },
+                { "text", i.ToString("0.00") }
+            };  
+
+            var feature = new Feature(mline, attributes);
+            features.Add(feature);
+        }
+
+        var fields = new List<DbfField>();
+        fields.AddDateField("date");
+        fields.AddFloatField("float");
+        fields.AddNumericInt32Field("int");
+        fields.AddLogicalField("logical");
+        fields.AddCharacterField("text");
+
+        var options = new ShapefileWriterOptions(ShapeType.PolyLine, fields.ToArray());
+        var shpPath = TestShapefiles.GetTempShpPath();
+        using (var shpWriter = Shapefile.OpenWrite(shpPath, options))
+        {
+            shpWriter.Write(features);
+        }
+        TestShapefiles.DeleteShp(shpPath);
+    }
+
+    [Test]
+    public void CreateShp_NullInt_Manual_ShpWriter()
+    {
+        var fields = new List<DbfField>();
+        var dateField = fields.AddDateField("date");
+        var floatField = fields.AddFloatField("float");
+        var intField = fields.AddNumericInt32Field("int");
+        var logicalField = fields.AddLogicalField("logical");
+        var textField = fields.AddCharacterField("text");
+
+        var options = new ShapefileWriterOptions(ShapeType.PolyLine, fields.ToArray());
+        var shpPath = TestShapefiles.GetTempShpPath();
+        using (var shpWriter = Shapefile.OpenWrite(shpPath, options))
+        {
+            for (var i = 1; i < 5; i++)
+            {
+                var lineCoords = new List<Coordinate>
+                {
+                    new(i, i + 1),
+                    new(i, i),
+                    new(i + 1, i)
+                };
+                var line = new LineString(lineCoords.ToArray());
+                var mline = new MultiLineString(new LineString[] { line });
+
+                int? nullableIntValue = null;
+
+                shpWriter.Geometry = mline;
+                dateField.DateValue = DateTime.Now;
+                floatField.NumericValue = i * 0.1;
+                intField.NumericValue = nullableIntValue;
+                logicalField.LogicalValue = i % 2 == 0;
+                textField.StringValue = i.ToString("0.00");
+                shpWriter.Write();
+            }
+        }
+        TestShapefiles.DeleteShp(shpPath);
+    }
+}

test/NetTopologySuite.IO.Esri.Test/Issues/Issue033.cs

@@ -0,0 +1,30 @@
+using NetTopologySuite.IO.Esri.Shapefiles.Readers;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NetTopologySuite.IO.Esri.Test.Issues;
+
+/// <summary>
+/// https://github.com/NetTopologySuite/NetTopologySuite.IO.Esri/issues/33
+/// </summary>
+internal class Issue033
+{
+    [Test]
+    public void WriteAllFeatures_NullAttributeValue()
+    {
+        string shpPath = TestShapefiles.PathTo(@"Issues/033/sample/ne_10m_admin_0_countries_fra.shp");
+        string shpCopyPath = TestShapefiles.GetTempShpPath();
+
+        var config = new ShapefileReaderOptions()
+        {
+            GeometryBuilderMode = GeometryBuilderMode.IgnoreInvalidShapes
+        };
+
+        var features = Shapefile.ReadAllFeatures(shpPath, config);
+        Shapefile.WriteAllFeatures(features, shpCopyPath);
+    }
+}