Fix dataframe incorrectly parse CSV when renameDuplicatedColumns is t…

…rue (dotnet#7242) * Fix dataframe incorrectly parse CSV when renameDuplicatedColumns is true * Update System.Numerics.Tensors dependency
michaelgsharp · Oct 9, 2024 · 1843bcd · 1843bcd
1 parent 9baf26b
commit 1843bcd
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs
@@ -388,7 +388,8 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
                 // First pass: schema and number of rows.
                 while ((fields = parser.ReadFields()) != null)
                 {
-                    if (renameDuplicatedColumns)
+                    //Only first row contains column names
+                    if (renameDuplicatedColumns && rowline == 0)
                     {
                         var names = new Dictionary<string, int>();
 

diff --git a/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj b/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj
@@ -46,7 +46,7 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
-    <PackageReference Include="System.Numerics.Tensors" Version="9.0.0-preview.6.24327.7" />
+    <PackageReference Include="System.Numerics.Tensors" Version="9.0.0-rc.1.24431.7" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETStandard'">

diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
@@ -119,6 +119,25 @@ private static Stream GetStream(string streamData)
             return new MemoryStream(Encoding.Default.GetBytes(streamData));
         }
 
+        [Fact]
+        public void TestReadCsvWithHeaderCultureInfoAndColumnTypeAutoGuess()
+        {
+            //see https://github.com/dotnet/machinelearning/issues/7240
+
+            CultureInfo.CurrentCulture = CultureInfo.InvariantCulture; // or en-US
+
+            string csv =
+@"""Col1"",""Col2"",""Col3"",""Col4""
+""v1.1"",""5/7/2017"",""v3.1"",""v4.1""
+"""","""",""v3.2"",""v4.2""
+";
+
+            var dataFrame = DataFrame.LoadCsvFromString(csv, separator: ',', header: true,
+                dataTypes: null, // guess the column types
+                renameDuplicatedColumns: true, // try to rename the duplicated columns, if any
+                cultureInfo: CultureInfo.InvariantCulture);
+        }
+
         [Theory]
         [InlineData(false)]
         [InlineData(true)]