From 8c4aedfb060e4fc022f0c389636cc0cf68a1e73c Mon Sep 17 00:00:00 2001 From: rAJt-11 Date: Fri, 24 Jul 2026 15:38:27 +0530 Subject: [PATCH 1/2] Add LoadFromDatabase convenience methods (#4182) Add LoadFromDatabase extension methods on DataOperationsCatalog that combine CreateDatabaseLoader() and Load(DatabaseSource) into a single call, mirroring the existing LoadFromTextFile sugar. --- .../Database/DatabaseLoaderCatalog.cs | 48 +++++++++++++++++++ .../Microsoft.ML.Tests/DatabaseLoaderTests.cs | 30 ++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCatalog.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCatalog.cs index 675f3f39ab..a3bcad6e2e 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCatalog.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCatalog.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Data.Common; using Microsoft.ML.Data; namespace Microsoft.ML @@ -38,5 +39,52 @@ public static DatabaseLoader CreateDatabaseLoader(this DataOperationsCatalog cat /// The catalog. public static DatabaseLoader CreateDatabaseLoader(this DataOperationsCatalog catalog) => DatabaseLoader.CreateDatabaseLoader(CatalogUtils.GetEnvironment(catalog)); + + /// + /// Load an from a database using a . + /// Note that 's are lazy, so no actual loading happens here, just schema validation. + /// + /// Defines the schema of the data to be loaded. Use public fields or properties + /// decorated with (and possibly other attributes) to specify the column + /// names and their data types in the schema of the loaded data. + /// The catalog. + /// The containing the connection and command information. + /// The data view. + public static IDataView LoadFromDatabase(this DataOperationsCatalog catalog, + DatabaseSource source) + => catalog.CreateDatabaseLoader().Load(source); + + /// + /// Load an from a database using a . + /// Note that 's are lazy, so no actual loading happens here, just schema validation. + /// + /// Defines the schema of the data to be loaded. Use public fields or properties + /// decorated with (and possibly other attributes) to specify the column + /// names and their data types in the schema of the loaded data. + /// The catalog. + /// The factory used to create the . + /// The string used to open the connection. + /// The text command to run against the data source. + /// The data view. + public static IDataView LoadFromDatabase(this DataOperationsCatalog catalog, + DbProviderFactory providerFactory, string connectionString, string commandText) + => catalog.LoadFromDatabase(new DatabaseSource(providerFactory, connectionString, commandText)); + + /// + /// Load an from a database using a . + /// Note that 's are lazy, so no actual loading happens here, just schema validation. + /// + /// Defines the schema of the data to be loaded. Use public fields or properties + /// decorated with (and possibly other attributes) to specify the column + /// names and their data types in the schema of the loaded data. + /// The catalog. + /// The factory used to create the . + /// The string used to open the connection. + /// The text command to run against the data source. + /// The timeout (in seconds) for the database command. + /// The data view. + public static IDataView LoadFromDatabase(this DataOperationsCatalog catalog, + DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeoutInSeconds) + => catalog.LoadFromDatabase(new DatabaseSource(providerFactory, connectionString, commandText, commandTimeoutInSeconds)); } } diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 2c70e5a9cb..91fb93b7e4 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -127,6 +127,36 @@ public void IrisLightGbmWithLoadColumnName() }).PredictedLabel); } + [LightGBMFact] + public void IrisVectorLightGbmUsingLoadFromDatabase() + { + var mlContext = new MLContext(seed: 1); + + var trainingData = mlContext.Data.LoadFromDatabase(GetIrisDatabaseSource("SELECT * FROM {0}")); + + IEstimator pipeline = mlContext.Transforms.Conversion.MapValueToKey("Label") + .Append(mlContext.Transforms.Concatenate("Features", "SepalInfo", "PetalInfo")) + .AppendCacheCheckpoint(mlContext) + .Append(mlContext.MulticlassClassification.Trainers.LightGbm()) + .Append(mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel")); + + var model = pipeline.Fit(trainingData); + + var engine = mlContext.Model.CreatePredictionEngine(model); + + Assert.Equal(0, engine.Predict(new IrisVectorData() + { + SepalInfo = new float[] { 4.5f, 5.6f }, + PetalInfo = new float[] { 0.5f, 0.5f }, + }).PredictedLabel); + + Assert.Equal(1, engine.Predict(new IrisVectorData() + { + SepalInfo = new float[] { 4.9f, 2.4f }, + PetalInfo = new float[] { 3.3f, 1.0f }, + }).PredictedLabel); + } + [LightGBMFact] public void IrisVectorLightGbm() { From 1d55c9d28dd7186cd3dd157c823ae17188f4104c Mon Sep 17 00:00:00 2001 From: rAJt-11 Date: Mon, 27 Jul 2026 10:56:48 +0530 Subject: [PATCH 2/2] Use a fast in-memory SQLite load test for LoadFromDatabase (#4182) Replace the redundant LightGBM training test with a focused, self-contained in-memory SQLite test that verifies LoadFromDatabase loads the expected data through both the DatabaseSource and DbProviderFactory overloads. This scopes the test to the new data-loading API and avoids adding a training run to the test suite. --- .../Microsoft.ML.Tests/DatabaseLoaderTests.cs | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 91fb93b7e4..32201d82e4 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -127,34 +127,51 @@ public void IrisLightGbmWithLoadColumnName() }).PredictedLabel); } - [LightGBMFact] - public void IrisVectorLightGbmUsingLoadFromDatabase() + [X86X64FactAttribute("The SQLite un-managed code, SQLite.interop, only supports x86/x64 architectures.")] + public void LoadFromDatabaseLoadsExpectedData() { - var mlContext = new MLContext(seed: 1); + // Use an in-memory SQLite database so the test is fast and self-contained. + var connectionString = "DataSource=LoadFromDatabaseTest;Mode=Memory;Version=3;Timeout=120;Cache=Shared"; + using (var connection = new SQLiteConnection(connectionString)) + { + connection.Open(); + using (var command = new SQLiteCommand(connection)) + { + command.CommandText = """ + BEGIN; + DROP TABLE IF EXISTS Iris; + CREATE TABLE IF NOT EXISTS Iris (Label INTEGER, SepalLength REAL, SepalWidth REAL, PetalLength REAL, PetalWidth REAL); + INSERT INTO Iris VALUES (0, 4.5, 5.6, 0.5, 0.5); + INSERT INTO Iris VALUES (1, 4.9, 2.4, 3.3, 1.0); + COMMIT; + """; + command.ExecuteNonQuery(); + } - var trainingData = mlContext.Data.LoadFromDatabase(GetIrisDatabaseSource("SELECT * FROM {0}")); + var mlContext = new MLContext(seed: 1); - IEstimator pipeline = mlContext.Transforms.Conversion.MapValueToKey("Label") - .Append(mlContext.Transforms.Concatenate("Features", "SepalInfo", "PetalInfo")) - .AppendCacheCheckpoint(mlContext) - .Append(mlContext.MulticlassClassification.Trainers.LightGbm()) - .Append(mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel")); + // LoadFromDatabase combines CreateDatabaseLoader() and Load(DatabaseSource) + // into a single call. Exercise both the DatabaseSource overload and the + // DbProviderFactory arguments overload; both should load identical data. + var source = new DatabaseSource(SQLiteFactory.Instance, connectionString, "SELECT * FROM Iris"); + var loadedFromSource = mlContext.Data.LoadFromDatabase(source); + var loadedFromArgs = mlContext.Data.LoadFromDatabase(SQLiteFactory.Instance, connectionString, "SELECT * FROM Iris"); - var model = pipeline.Fit(trainingData); + foreach (var data in new[] { loadedFromSource, loadedFromArgs }) + { + var rows = mlContext.Data.CreateEnumerable(data, reuseRowObject: false).ToArray(); - var engine = mlContext.Model.CreatePredictionEngine(model); + rows.Length.Should().Be(2); - Assert.Equal(0, engine.Predict(new IrisVectorData() - { - SepalInfo = new float[] { 4.5f, 5.6f }, - PetalInfo = new float[] { 0.5f, 0.5f }, - }).PredictedLabel); + rows[0].Label.Should().Be(0); + rows[0].SepalInfo.Should().Equal(4.5f, 5.6f); + rows[0].PetalInfo.Should().Equal(0.5f, 0.5f); - Assert.Equal(1, engine.Predict(new IrisVectorData() - { - SepalInfo = new float[] { 4.9f, 2.4f }, - PetalInfo = new float[] { 3.3f, 1.0f }, - }).PredictedLabel); + rows[1].Label.Should().Be(1); + rows[1].SepalInfo.Should().Equal(4.9f, 2.4f); + rows[1].PetalInfo.Should().Equal(3.3f, 1.0f); + } + } } [LightGBMFact]