fixing KeepDuplicates, adding a Hashtable for performance reasons and squashing due to a weird rebase

Author: SimaTian

src/Build/BackEnd/Components/RequestBuilder/IntrinsicTasks/ItemGroupIntrinsicTask.cs

@@ -215,21 +215,28 @@ private void ExecuteAdd(ProjectItemGroupTaskItemInstance child, ItemBucket bucke
                 FileSystems.Default,
                 LoggingContext);
 
+            Action<List<ProjectItemInstance>> logFunction = null;
+
             if (LogTaskInputs && !LoggingContext.LoggingService.OnlyLogCriticalEvents && itemsToAdd?.Count > 0)
             {
-                ItemGroupLoggingHelper.LogTaskParameter(
-                    LoggingContext,
-                    TaskParameterMessageKind.AddItem,
-                    parameterName: null,
-                    propertyName: null,
-                    child.ItemType,
-                    itemsToAdd,
-                    logItemMetadata: true,
-                    child.Location);
+                logFunction = (itemList) =>
+                {
+                    ItemGroupLoggingHelper.LogTaskParameter(
+                        LoggingContext,
+                        TaskParameterMessageKind.AddItem,
+                        parameterName: null,
+                        propertyName: null,
+                        child.ItemType,
+                        itemList,
+                        logItemMetadata: true,
+                        child.Location);
+                };
             }
+           
 
-            // Now add the items we created to the lookup.
-            bucket.Lookup.AddNewItemsOfItemType(child.ItemType, itemsToAdd, !keepDuplicates); // Add in one operation for potential copy-on-write
+                // Now add the items we created to the lookup.
+                bucket.Lookup.AddNewItemsOfItemType(child.ItemType, itemsToAdd, !keepDuplicates, logFunction);
+            // Add in one operation for potential copy-on-write
         }
 
         /// <summary>

src/Build/BackEnd/Components/RequestBuilder/Lookup.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using Microsoft.Build.Collections;
 using Microsoft.Build.Evaluation;
@@ -636,7 +637,7 @@ internal void SetProperty(ProjectPropertyInstance property)
         /// <summary>
         /// Implements a true add, an item that has been created in a batch.
         /// </summary>
-        internal void AddNewItemsOfItemType(string itemType, ICollection<ProjectItemInstance> group, bool doNotAddDuplicates = false)
+        internal void AddNewItemsOfItemType(string itemType, ICollection<ProjectItemInstance> group, bool doNotAddDuplicates = false, Action<List<ProjectItemInstance>> logFunction = null)
         {
             // Adding to outer scope could be easily implemented, but our code does not do it at present
             MustNotBeOuterScope();
@@ -663,12 +664,32 @@ internal void AddNewItemsOfItemType(string itemType, ICollection<ProjectItemInst
 
                 // Ensure we don't also add any that already exist.
                 var existingItems = GetItems(itemType);
+              
+                var existingItemsHashSet = existingItems.ToHashSet(ProjectItemInstance.EqualityComparer);
+                
                 if (existingItems.Count > 0)
                 {
-                    itemsToAdd = itemsToAdd.Where(item => !existingItems.Contains(item, ProjectItemInstance.EqualityComparer));
+                    
+                    var deduplicatedItemsToAdd = new List<ProjectItemInstance>();
+                    foreach (var item in itemsToAdd) {
+                        if (!existingItemsHashSet.Contains(item)) {
+                            deduplicatedItemsToAdd.Add(item);
+                        }
+                    }
+                   itemsToAdd = deduplicatedItemsToAdd;
                 }
             }
 
+            if (doNotAddDuplicates)
+            {
+                logFunction?.Invoke(itemsToAdd.ToList());
+            }
+            else {
+                var groupAsList = group as List<ProjectItemInstance>;
+                logFunction?.Invoke(groupAsList ?? group.ToList());  
+            }
+           
+
             PrimaryAddTable.ImportItemsOfType(itemType, itemsToAdd);
         }