From 1a4798c5a48011fa951550ff314ecc2a1873a1c4 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:04:37 -0700 Subject: [PATCH 01/25] Rename to ReferencingColumns --- .../Configurations/SqlConfigValidatorExceptions.cs | 2 +- DataGateway.Service/Configurations/SqlConfigValidatorMain.cs | 4 ++-- DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs | 4 ++-- DataGateway.Service/Models/DatabaseSchema.cs | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DataGateway.Service/Configurations/SqlConfigValidatorExceptions.cs b/DataGateway.Service/Configurations/SqlConfigValidatorExceptions.cs index 1674a841ec..42e8ddbae1 100644 --- a/DataGateway.Service/Configurations/SqlConfigValidatorExceptions.cs +++ b/DataGateway.Service/Configurations/SqlConfigValidatorExceptions.cs @@ -313,7 +313,7 @@ private void ValidateRefColumnsExistInRefTable(List referencedColumns, s /// private void ValidateFKColumnsHaveMatchingTableColumns(ForeignKeyDefinition foreignKey, TableDefinition table) { - IEnumerable unmatchedFkCols = foreignKey.Columns.Except(table.Columns.Keys); + IEnumerable unmatchedFkCols = foreignKey.ReferencingColumns.Except(table.Columns.Keys); if (unmatchedFkCols.Any()) { diff --git a/DataGateway.Service/Configurations/SqlConfigValidatorMain.cs b/DataGateway.Service/Configurations/SqlConfigValidatorMain.cs index 1d8a27eda1..a909f5fbdd 100644 --- a/DataGateway.Service/Configurations/SqlConfigValidatorMain.cs +++ b/DataGateway.Service/Configurations/SqlConfigValidatorMain.cs @@ -182,8 +182,8 @@ private void ValidateForeignKeyColumns(ForeignKeyDefinition foreignKey, TableDef if (HasExplicitColumns(foreignKey)) { - ValidateNoDuplicateFkColumns(foreignKey.Columns, refColumns: false); - columns = foreignKey.Columns; + ValidateNoDuplicateFkColumns(foreignKey.ReferencingColumns, refColumns: false); + columns = foreignKey.ReferencingColumns; ValidateFKColumnsHaveMatchingTableColumns(foreignKey, table); } else diff --git a/DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs b/DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs index f044212572..8d6295f423 100644 --- a/DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs +++ b/DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs @@ -227,7 +227,7 @@ private static bool TableHasForeignKey(TableDefinition table) /// private static bool HasExplicitColumns(ForeignKeyDefinition fk) { - return fk.Columns.Count > 0; + return fk.ReferencingColumns.Count > 0; } /// @@ -494,7 +494,7 @@ private static IEnumerable GetPkAndFkColumns(TableDefinition table) foreach (KeyValuePair nameFKPair in table.ForeignKeys) { ForeignKeyDefinition foreignKey = nameFKPair.Value; - columns.AddRange(foreignKey.Columns); + columns.AddRange(foreignKey.ReferencingColumns); } return columns; diff --git a/DataGateway.Service/Models/DatabaseSchema.cs b/DataGateway.Service/Models/DatabaseSchema.cs index b46bc28620..38df1fb18a 100644 --- a/DataGateway.Service/Models/DatabaseSchema.cs +++ b/DataGateway.Service/Models/DatabaseSchema.cs @@ -50,10 +50,10 @@ public class ForeignKeyDefinition /// /// The list of columns of the table that make up the foreign key. - /// If this list is empty, the primary key columns of the + /// If this list is empty, the primary key columns of the referencing /// table are implicitly assumed to be the foreign key columns. /// - public List Columns { get; set; } = new(); + public List ReferencingColumns { get; set; } = new(); } public class AuthorizationRule From b6513b86b33a325770662560cbbd00df3e0b4821 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:05:00 -0700 Subject: [PATCH 02/25] Rename to Referencing Columns --- .../Resolvers/Sql Query Structures/SqlQueryStructure.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataGateway.Service/Resolvers/Sql Query Structures/SqlQueryStructure.cs b/DataGateway.Service/Resolvers/Sql Query Structures/SqlQueryStructure.cs index e4d52157c5..a380e14bf9 100644 --- a/DataGateway.Service/Resolvers/Sql Query Structures/SqlQueryStructure.cs +++ b/DataGateway.Service/Resolvers/Sql Query Structures/SqlQueryStructure.cs @@ -619,7 +619,7 @@ void AddGraphQLFields(IReadOnlyList Selections) /// private static List GetFkColumns(ForeignKeyDefinition fk, TableDefinition table) { - return fk.Columns.Count > 0 ? fk.Columns : table.PrimaryKey; + return fk.ReferencingColumns.Count > 0 ? fk.ReferencingColumns : table.PrimaryKey; } /// From e49672ceab9eb5de208d7c80e36d864a28ecff26 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:05:42 -0700 Subject: [PATCH 03/25] BuildForeignKeyQuery --- .../Resolvers/BaseSqlQueryBuilder.cs | 27 +++++++++++++++++++ .../Resolvers/IQueryBuilder.cs | 6 +++++ .../Resolvers/MySqlQueryBuilder.cs | 23 ++++++++++++++++ .../Resolvers/PostgresQueryBuilder.cs | 27 +++++++++++++++++++ 4 files changed, 83 insertions(+) diff --git a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs index 2c7e3bc79c..55d3f3df1f 100644 --- a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs @@ -259,5 +259,32 @@ public string JoinPredicateStrings(params string?[] predicateStrings) return string.Join(" AND ", validPredicates); } + + /// + public virtual string BuildForeignKeyQuery(string schemaName, string tableName) + { + return $"" + + $"SELECT " + + $"ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))} " + + $"ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, " + + $"ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, " + + $"ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} " + + $"FROM " + + $"INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints " + + $"INNER JOIN " + + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage " + + $"ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG " + + $"AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA " + + $"AND ReferentialConstraints.CONSTRAINT_NAME = ReferencingColumnUsage.CONSTRAINT_NAME " + + $"INNER JOIN " + + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencedColumnUsage " + + $"ON ReferentialConstraints.UNIQUE_CONSTRAINT_CATALOG = ReferencedColumnUsage.CONSTRAINT_CATALOG " + + $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA " + + $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME " + + $"WHERE " + + $"ReferencingColumnUsage.SCHEMA_NAME = {QuoteIdentifier($"@{nameof(schemaName)}")} " + + $"AND ReferencingColumnUsage.TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + + $"AND ReferencingColumnUsage.ORDINAL_POSITION = ReferencedColumnUsage.ORDINAL_POSITION;"; + } } } diff --git a/DataGateway.Service/Resolvers/IQueryBuilder.cs b/DataGateway.Service/Resolvers/IQueryBuilder.cs index c18c2ee831..de27d6324d 100644 --- a/DataGateway.Service/Resolvers/IQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/IQueryBuilder.cs @@ -35,5 +35,11 @@ public interface IQueryBuilder /// query. /// public string Build(SqlUpsertQueryStructure structure); + + /// + /// Builds the query to obtain foreign key information for the given + /// schema, table name. + /// + public string BuildForeignKeyQuery(string schemaName, string tableName); } } diff --git a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs index b8d3c1f329..261610a353 100644 --- a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs @@ -145,6 +145,29 @@ public string Build(SqlUpsertQueryStructure structure) } } + /// + /// For MySql, the table name is only a 2 part name. + /// The schema name passed here is actually the database + /// from the connection string and that is compared against the SCHEMA_NAME + /// in the view KEY_COLUMN_USAGE. + /// + public override string BuildForeignKeyQuery(string databaseName, string tableName) + { + return $"" + + $"SELECT " + + $"CONSTRAINT_NAME {QuoteIdentifier("Foreign Key Name")}" + + $"COLUMN_NAME {QuoteIdentifier("Referencing Column")}, " + + $"REFERENCED_TABLE_NAME {QuoteIdentifier("Referenced Table")}, " + + $"REFERENCED_COLUMN_NAME {QuoteIdentifier("Referenced Column")} " + + $"FROM " + + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + + $"WHERE " + + $"SCHEMA_NAME = {QuoteIdentifier($"@{nameof(databaseName)}")} " + + $"AND TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + + $"AND REFERENCED_TABLE_NAME IS NOT NULL " + + $"AND REFERENCED_COLUMN_NAME IS NOT NULL;"; + } + /// /// Makes the query segments to store PK during an update /// diff --git a/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs b/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs index 648efedac5..3fdf2bce21 100644 --- a/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Data.Common; using System.Linq; using System.Text; @@ -85,6 +86,32 @@ public string Build(SqlUpsertQueryStructure structure) throw new NotImplementedException(); } + /// + public string BuildForeignKeyQuery(string schemaName, string tableName) + { + return $"" + + $"SELECT " + + $"ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier("Referencing Column")}, " + + $"ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier("Referenced Table")}, " + + $"ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier("Referenced Column")} " + + $"FROM " + + $"INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints " + + $"INNER JOIN " + + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage " + + $"ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG " + + $"AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA " + + $"AND ReferentialConstraints.CONSTRAINT_NAME = ReferencingColumnUsage.CONSTRAINT_NAME " + + $"INNER JOIN " + + $"INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ReferencedColumnUsage " + + $"ON ReferentialConstraints.UNIQUE_CONSTRAINT_CATALOG = ReferencedColumnUsage.CONSTRAINT_CATALOG " + + $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA " + + $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME " + + $"WHERE " + + $"ReferencingColumnUsage.SCHEMA_NAME = {QuoteIdentifier($"@{nameof(schemaName)}")} " + + $"AND ReferencingColumnUsage.TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + + $"AND ReferencingColumnUsage.POSITION_IN_UNIQUE_CONSTRAINT IS NOT NULL;"; + } + /// protected override string Build(KeysetPaginationPredicate predicate) { From ba42ff1e7a22dbb427333b97274515b0cef6bab4 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:06:00 -0700 Subject: [PATCH 04/25] PopulateForeignKeyDefinitionAsync --- .../MsSqlMetadataProvider.cs | 8 +- .../MySqlMetadataProvider.cs | 33 +++++++- .../PostgreSqlMetadataProvider.cs | 8 +- .../MetadataProviders/SqlMetadataProvider.cs | 81 ++++++++++++++++++- 4 files changed, 123 insertions(+), 7 deletions(-) diff --git a/DataGateway.Service/Services/MetadataProviders/MsSqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/MsSqlMetadataProvider.cs index c0060c0148..8c3981f066 100644 --- a/DataGateway.Service/Services/MetadataProviders/MsSqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/MsSqlMetadataProvider.cs @@ -1,4 +1,5 @@ using Azure.DataGateway.Service.Configurations; +using Azure.DataGateway.Service.Resolvers; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Options; @@ -13,8 +14,11 @@ namespace Azure.DataGateway.Service.Services public class MsSqlMetadataProvider : SqlMetadataProvider { - public MsSqlMetadataProvider(IOptions dataGatewayConfig) - : base(dataGatewayConfig) + public MsSqlMetadataProvider( + IOptions dataGatewayConfig, + IQueryExecutor queryExecutor, + IQueryBuilder sqlQueryBuilder) + : base(dataGatewayConfig, queryExecutor, sqlQueryBuilder) { } diff --git a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs index 7d3b365dc5..9b8801ed38 100644 --- a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Azure.DataGateway.Service.Configurations; +using Azure.DataGateway.Service.Resolvers; using Microsoft.Extensions.Options; using MySqlConnector; @@ -13,8 +14,11 @@ namespace Azure.DataGateway.Service.Services /// public class MySqlMetadataProvider : SqlMetadataProvider, ISqlMetadataProvider { - public MySqlMetadataProvider(IOptions dataGatewayConfig) - : base(dataGatewayConfig) + public MySqlMetadataProvider( + IOptions dataGatewayConfig, + IQueryExecutor queryExecutor, + IQueryBuilder sqlQueryBuilder) + : base(dataGatewayConfig, queryExecutor, sqlQueryBuilder) { } @@ -52,5 +56,30 @@ protected override async Task GetColumnsAsync( return allColumns; } + + /// + /// For MySql, the table name is only a 2 part name. + /// The database name from the connection string needs to be used instead of schemaName. + /// + protected override string GetForeignKeyQuery(string schemaName, string tableName) + { + using MySqlConnection conn = new(ConnectionString); + return SqlQueryBuilder!.BuildForeignKeyQuery(conn.Database, tableName); + } + + /// + /// For MySql, the table name is only a 2 part name. + /// The database name from the connection string needs to be used instead of schemaName. + /// + protected override Dictionary + GetForeignKeyQueryParams(string schemaName, string tableName) + { + using MySqlConnection conn = new(ConnectionString); + string databaseName = conn.Database; + Dictionary parameters = new(); + parameters.Add(nameof(databaseName), databaseName); + parameters.Add(nameof(tableName), tableName); + return parameters; + } } } diff --git a/DataGateway.Service/Services/MetadataProviders/PostgreSqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/PostgreSqlMetadataProvider.cs index 3610fcd9ab..cb9bab262c 100644 --- a/DataGateway.Service/Services/MetadataProviders/PostgreSqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/PostgreSqlMetadataProvider.cs @@ -1,4 +1,5 @@ using Azure.DataGateway.Service.Configurations; +using Azure.DataGateway.Service.Resolvers; using Microsoft.Extensions.Options; using Npgsql; @@ -13,8 +14,11 @@ namespace Azure.DataGateway.Service.Services public class PostgreSqlMetadataProvider : SqlMetadataProvider { - public PostgreSqlMetadataProvider(IOptions dataGatewayConfig) - : base(dataGatewayConfig) + public PostgreSqlMetadataProvider( + IOptions dataGatewayConfig, + IQueryExecutor queryExecutor, + IQueryBuilder sqlQueryBuilder) + : base(dataGatewayConfig, queryExecutor, sqlQueryBuilder) { } } diff --git a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs index 36d311b191..17803afac8 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Azure.DataGateway.Service.Configurations; using Azure.DataGateway.Service.Models; +using Azure.DataGateway.Service.Resolvers; using Microsoft.Extensions.Options; namespace Azure.DataGateway.Service.Services @@ -20,18 +21,29 @@ public class SqlMetadataProvider : ISqlMeta where DataAdapterT : DbDataAdapter, new() where CommandT : DbCommand, new() { + // nullable since Mock tests do not need it. + private readonly IQueryExecutor? _queryExecutor; + private const int NUMBER_OF_RESTRICTIONS = 4; protected const string TABLE_TYPE = "BASE TABLE"; protected string ConnectionString { get; init; } + // nullable since Mock tests don't need this. + protected IQueryBuilder? SqlQueryBuilder { get; init; } + protected DataSet EntitiesDataSet { get; init; } - public SqlMetadataProvider(IOptions dataGatewayConfig) + public SqlMetadataProvider( + IOptions dataGatewayConfig, + IQueryExecutor queryExecutor, + IQueryBuilder queryBuilder) { ConnectionString = dataGatewayConfig.Value.DatabaseConnection.ConnectionString; EntitiesDataSet = new(); + SqlQueryBuilder = queryBuilder; + _queryExecutor = queryExecutor; } /// @@ -76,6 +88,11 @@ public virtual async Task PopulateTableDefinitionAsync( PopulateColumnDefinitionWithHasDefault( tableDefinition, columnsInTable); + + await PopulateForeignKeyDefinitionAsync( + schemaName, + tableName, + tableDefinition); } /// @@ -175,5 +192,67 @@ protected void PopulateColumnDefinitionWithHasDefault( } } } + + protected async Task PopulateForeignKeyDefinitionAsync( + string schemaName, + string tableName, + TableDefinition tableDefinition) + { + string queryForForeignKeyInfo = GetForeignKeyQuery(schemaName, tableName); + Dictionary parameters = + GetForeignKeyQueryParams(schemaName, tableName); + using DbDataReader reader = + await _queryExecutor!.ExecuteQueryAsync(queryForForeignKeyInfo, parameters); + + Dictionary? foreignKeyInfo = + await _queryExecutor!.ExtractRowFromDbDataReader(reader); + + while (foreignKeyInfo != null) + { + string foreignKeyName = (string)foreignKeyInfo[nameof(ForeignKeyDefinition)]!; + ForeignKeyDefinition? foreignKeyDefinition; + if (!tableDefinition.ForeignKeys.TryGetValue(foreignKeyName, out foreignKeyDefinition)) + { + foreignKeyDefinition = new(); + foreignKeyDefinition.ReferencedTable = + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedTable)]!; + } + + foreignKeyDefinition.ReferencedColumns.Add( + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedColumns)]!); + foreignKeyDefinition.ReferencingColumns.Add( + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencingColumns)]!); + + foreignKeyInfo = await _queryExecutor.ExtractRowFromDbDataReader(reader); + } + } + + /// + /// Invokes the underlying query builder to + /// build the query useful for retrieving the foreign key information. + /// + /// + /// + /// The Sql query to use. + protected virtual string GetForeignKeyQuery(string schemaName, string tableName) + { + return SqlQueryBuilder!.BuildForeignKeyQuery(schemaName, tableName); + } + + /// + /// Builds the dictionary of parameters and their values required for the + /// foreign key query. + /// + /// + /// + /// The dictionary populated with parameters. + protected virtual Dictionary + GetForeignKeyQueryParams(string schemaName, string tableName) + { + Dictionary parameters = new(); + parameters.Add(nameof(schemaName), schemaName); + parameters.Add(nameof(tableName), tableName); + return parameters; + } } } From 545364ac42a62688531fe2a2e778605f63468624 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:19:03 -0700 Subject: [PATCH 05/25] Add multiple column foreign key reference Add non-key column foreign key --- DataGateway.Service/MsSqlBooks.sql | 28 +++++++++++++++++++++--- DataGateway.Service/MySqlBooks.sql | 29 +++++++++++++++++++++---- DataGateway.Service/PostgreSqlBooks.sql | 28 +++++++++++++++++++++--- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/DataGateway.Service/MsSqlBooks.sql b/DataGateway.Service/MsSqlBooks.sql index 39d4a20481..1340309492 100644 --- a/DataGateway.Service/MsSqlBooks.sql +++ b/DataGateway.Service/MsSqlBooks.sql @@ -7,6 +7,7 @@ DROP TABLE IF EXISTS publishers; DROP TABLE IF EXISTS magazines; DROP TABLE IF EXISTS comics; DROP TABLE IF EXISTS stocks; +DROP TABLE IF EXISTS stocks_price; --Autogenerated id seed are set at 5001 for consistency with Postgres --This allows for tests using the same id values for both languages @@ -57,18 +58,27 @@ CREATE TABLE magazines( CREATE TABLE comics( id bigint PRIMARY KEY, title varchar(max) NOT NULL, - volume bigint IDENTITY(5001,1) + volume bigint IDENTITY(5001,1), + categoryName varchar(100) NOT NULL UNIQUE ); CREATE TABLE stocks( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - categoryName varchar(max) NOT NULL, + categoryName varchar(100) NOT NULL, piecesAvailable bigint DEFAULT 0, piecesRequired bigint DEFAULT 0 NOT NULL, PRIMARY KEY(categoryid,pieceid) ); +CREATE TABLE stocks_price( + categoryid bigint NOT NULL, + pieceid bigint NOT NULL, + instant timestamp, + price float, + PRIMARY KEY(categoryid, pieceid, instant) +); + ALTER TABLE books ADD CONSTRAINT book_publisher_fk FOREIGN KEY (publisher_id) @@ -99,6 +109,18 @@ FOREIGN KEY (author_id) REFERENCES authors (id) ON DELETE CASCADE; +ALTER TABLE stocks +ADD CONSTRAINT stocks_comics_fk +FOREIGN KEY (categoryName) +REFERENCES comics (categoryName) +ON DELETE CASCADE; + +ALTER TABLE stocks_price +ADD CONSTRAINT stocks_price_stocks_fk +FOREIGN KEY (categoryid, pieceid) +REFERENCES stocks (categoryid, pieceid) +ON DELETE CASCADE; + SET IDENTITY_INSERT publishers ON INSERT INTO publishers(id, name) VALUES (1234, 'Big Company'), (2345, 'Small Town Publisher'), (2323, 'TBD Publishing One'), (2324, 'TBD Publishing Two Ltd'); SET IDENTITY_INSERT publishers OFF @@ -121,4 +143,4 @@ SET IDENTITY_INSERT reviews ON INSERT INTO reviews(id, book_id, content) VALUES (567, 1, 'Indeed a great book'), (568, 1, 'I loved it'), (569, 1, 'best book I read in years'); SET IDENTITY_INSERT reviews OFF -INSERT INTO stocks(categoryid, pieceid,categoryName) VALUES (1, 1, 'books'), (2, 1, 'magazines'); +INSERT INTO stocks(categoryid, pieceid, categoryName) VALUES (1, 1, 'books'), (2, 1, 'magazines'); diff --git a/DataGateway.Service/MySqlBooks.sql b/DataGateway.Service/MySqlBooks.sql index dcd25c74f8..52bf3d9707 100644 --- a/DataGateway.Service/MySqlBooks.sql +++ b/DataGateway.Service/MySqlBooks.sql @@ -5,8 +5,9 @@ DROP TABLE IF EXISTS book_website_placements; DROP TABLE IF EXISTS books; DROP TABLE IF EXISTS publishers; DROP TABLE IF EXISTS magazines; -DROP TABLE IF EXISTS comics; +DROP TABLE IF EXISTS stocks_price; DROP TABLE IF EXISTS stocks; +DROP TABLE IF EXISTS comics; CREATE TABLE publishers( id bigint AUTO_INCREMENT PRIMARY KEY, @@ -54,18 +55,27 @@ CREATE TABLE magazines( CREATE TABLE comics( id bigint PRIMARY KEY, title text NOT NULL, - volume bigint AUTO_INCREMENT UNIQUE KEY + volume bigint AUTO_INCREMENT UNIQUE KEY, + categoryName varchar(100) NOT NULL UNIQUE ); CREATE TABLE stocks( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - categoryName text NOT NULL, + categoryName varchar(100) NOT NULL, piecesAvailable bigint DEFAULT (0), piecesRequired bigint DEFAULT (0) NOT NULL, PRIMARY KEY(categoryid,pieceid) ); +CREATE TABLE stocks_price( + categoryid bigint NOT NULL, + pieceid bigint NOT NULL, + instant timestamp, + price float, + PRIMARY KEY(categoryid, pieceid, instant) +); + ALTER TABLE books ADD CONSTRAINT book_publisher_fk FOREIGN KEY (publisher_id) @@ -96,6 +106,18 @@ FOREIGN KEY (author_id) REFERENCES authors (id) ON DELETE CASCADE; +ALTER TABLE stocks +ADD CONSTRAINT stocks_comics_fk +FOREIGN KEY (categoryName) +REFERENCES comics (categoryName) +ON DELETE CASCADE; + +ALTER TABLE stocks_price +ADD CONSTRAINT stocks_price_stocks_fk +FOREIGN KEY (categoryid, pieceid) +REFERENCES stocks (categoryid, pieceid) +ON DELETE CASCADE; + INSERT INTO publishers(id, name) VALUES (1234, 'Big Company'), (2345, 'Small Town Publisher'), (2323, 'TBD Publishing One'), (2324, 'TBD Publishing Two Ltd'); INSERT INTO authors(id, name, birthdate) VALUES (123, 'Jelte', '2001-01-01'), (124, 'Aniruddh', '2002-02-02'); INSERT INTO books(id, title, publisher_id) VALUES (1, 'Awesome book', 1234), (2, 'Also Awesome book', 1234), (3, 'Great wall of china explained', 2345), (4, 'US history in a nutshell', 2345), (5, 'Chernobyl Diaries', 2323), (6, 'The Palace Door', 2324), (7, 'The Groovy Bar', 2324), (8, 'Time to Eat', 2324); @@ -112,4 +134,3 @@ ALTER TABLE publishers AUTO_INCREMENT = 5001; ALTER TABLE authors AUTO_INCREMENT = 5001; ALTER TABLE reviews AUTO_INCREMENT = 5001; ALTER TABLE comics AUTO_INCREMENT = 5001 - diff --git a/DataGateway.Service/PostgreSqlBooks.sql b/DataGateway.Service/PostgreSqlBooks.sql index 95e0f73e37..7bd2a3b19e 100644 --- a/DataGateway.Service/PostgreSqlBooks.sql +++ b/DataGateway.Service/PostgreSqlBooks.sql @@ -5,8 +5,9 @@ DROP TABLE IF EXISTS book_website_placements; DROP TABLE IF EXISTS books; DROP TABLE IF EXISTS publishers; DROP TABLE IF EXISTS magazines; -DROP TABLE IF EXISTS comics; +DROP TABLE IF EXISTS stocks_price; DROP TABLE IF EXISTS stocks; +DROP TABLE IF EXISTS comics; CREATE TABLE publishers( id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, @@ -53,18 +54,27 @@ CREATE TABLE magazines( CREATE TABLE comics( id bigint PRIMARY KEY, title text NOT NULL, - volume bigint GENERATED BY DEFAULT AS IDENTITY + volume bigint GENERATED BY DEFAULT AS IDENTITY, + categoryName varchar(100) NOT NULL UNIQUE ); CREATE TABLE stocks( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - categoryName text NOT NULL, + categoryName varchar(100) NOT NULL, piecesAvailable bigint DEFAULT 0, piecesRequired bigint DEFAULT 0 NOT NULL, PRIMARY KEY(categoryid,pieceid) ); +CREATE TABLE stocks_price( + categoryid bigint NOT NULL, + pieceid bigint NOT NULL, + instant timestamp, + price float, + PRIMARY KEY(categoryid, pieceid, instant) +); + ALTER TABLE books ADD CONSTRAINT book_publisher_fk FOREIGN KEY (publisher_id) @@ -95,6 +105,18 @@ FOREIGN KEY (author_id) REFERENCES authors (id) ON DELETE CASCADE; +ALTER TABLE stocks +ADD CONSTRAINT stocks_comics_fk +FOREIGN KEY (categoryName) +REFERENCES comics (categoryName) +ON DELETE CASCADE; + +ALTER TABLE stocks_price +ADD CONSTRAINT stocks_price_stocks_fk +FOREIGN KEY (categoryid, pieceid) +REFERENCES stocks (categoryid, pieceid) +ON DELETE CASCADE; + INSERT INTO publishers(id, name) VALUES (1234, 'Big Company'), (2345, 'Small Town Publisher'), (2323, 'TBD Publishing One'), (2324, 'TBD Publishing Two Ltd'); INSERT INTO authors(id, name, birthdate) VALUES (123, 'Jelte', '2001-01-01'), (124, 'Aniruddh', '2002-02-02'); INSERT INTO books(id, title, publisher_id) VALUES (1, 'Awesome book', 1234), (2, 'Also Awesome book', 1234), (3, 'Great wall of china explained', 2345), (4, 'US history in a nutshell', 2345), (5, 'Chernobyl Diaries', 2323), (6, 'The Palace Door', 2324), (7, 'The Groovy Bar', 2324), (8, 'Time to Eat', 2324); From b60c6a5abb5fbb70bc56046badd2e9432bf40140 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:22:30 -0700 Subject: [PATCH 06/25] Fix build errors --- .../SqlTests/SqlTestBase.cs | 16 +++++------ .../Resolvers/PostgresQueryBuilder.cs | 27 ------------------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs b/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs index 721378f77a..e4f88808af 100644 --- a/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs +++ b/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs @@ -60,30 +60,28 @@ protected static async Task InitializeTestFixture(TestContext context, string te { case TestCategory.POSTGRESQL: _queryBuilder = new PostgresQueryBuilder(); - _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( - config, - new PostgreSqlMetadataProvider(config)); _defaultSchemaName = "public"; _dbExceptionParser = new PostgresDbExceptionParser(); _queryExecutor = new QueryExecutor(config, _dbExceptionParser); + _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( + config, new PostgreSqlMetadataProvider(config, _queryExecutor, _queryBuilder)); break; case TestCategory.MSSQL: _queryBuilder = new MsSqlQueryBuilder(); - _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( - config, - new MsSqlMetadataProvider(config)); _defaultSchemaName = "dbo"; _dbExceptionParser = new DbExceptionParserBase(); _queryExecutor = new QueryExecutor(config, _dbExceptionParser); + _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( + config, + new MsSqlMetadataProvider(config, _queryExecutor, _queryBuilder)); break; case TestCategory.MYSQL: _queryBuilder = new MySqlQueryBuilder(); - _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( - config, - new MySqlMetadataProvider(config)); _defaultSchemaName = "mysql"; _dbExceptionParser = new MySqlDbExceptionParser(); _queryExecutor = new QueryExecutor(config, _dbExceptionParser); + _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( + config, new MySqlMetadataProvider(config, _queryExecutor, _queryBuilder)); break; } diff --git a/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs b/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs index 3fdf2bce21..648efedac5 100644 --- a/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/PostgresQueryBuilder.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Data.Common; using System.Linq; using System.Text; @@ -86,32 +85,6 @@ public string Build(SqlUpsertQueryStructure structure) throw new NotImplementedException(); } - /// - public string BuildForeignKeyQuery(string schemaName, string tableName) - { - return $"" + - $"SELECT " + - $"ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier("Referencing Column")}, " + - $"ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier("Referenced Table")}, " + - $"ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier("Referenced Column")} " + - $"FROM " + - $"INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints " + - $"INNER JOIN " + - $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage " + - $"ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG " + - $"AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA " + - $"AND ReferentialConstraints.CONSTRAINT_NAME = ReferencingColumnUsage.CONSTRAINT_NAME " + - $"INNER JOIN " + - $"INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE ReferencedColumnUsage " + - $"ON ReferentialConstraints.UNIQUE_CONSTRAINT_CATALOG = ReferencedColumnUsage.CONSTRAINT_CATALOG " + - $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA " + - $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME " + - $"WHERE " + - $"ReferencingColumnUsage.SCHEMA_NAME = {QuoteIdentifier($"@{nameof(schemaName)}")} " + - $"AND ReferencingColumnUsage.TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + - $"AND ReferencingColumnUsage.POSITION_IN_UNIQUE_CONSTRAINT IS NOT NULL;"; - } - /// protected override string Build(KeysetPaginationPredicate predicate) { From a6da0ccd3a98fddc72507ec267fc5e4b78f791ea Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Mon, 11 Apr 2022 19:25:52 -0700 Subject: [PATCH 07/25] Remove foreign key information from runtime(resolver) config file sql-config.json --- DataGateway.Service/sql-config.json | 31 ----------------------------- 1 file changed, 31 deletions(-) diff --git a/DataGateway.Service/sql-config.json b/DataGateway.Service/sql-config.json index aa03707c9e..295bbe918a 100644 --- a/DataGateway.Service/sql-config.json +++ b/DataGateway.Service/sql-config.json @@ -158,12 +158,6 @@ } }, "books": { - "ForeignKeys": { - "book_publisher_fk": { - "ReferencedTable": "publishers", - "Columns": [ "publisher_id" ] - } - }, "HttpVerbs": { "GET": { "AuthorizationType": "Authenticated" @@ -183,13 +177,6 @@ } }, "book_website_placements": { - "PrimaryKey": [ "id" ], - "ForeignKeys": { - "book_website_placement_book_fk": { - "ReferencedTable": "books", - "Columns": [ "book_id" ] - } - }, "HttpVerbs": { "GET": { "AuthorizationType": "Anonymous" @@ -216,12 +203,6 @@ } }, "reviews": { - "ForeignKeys": { - "review_book_fk": { - "ReferencedTable": "books", - "Columns": [ "book_id" ] - } - }, "HttpVerbs": { "GET": { "AuthorizationType": "Anonymous" @@ -229,20 +210,8 @@ } }, "book_author_link": { - "ForeignKeys": { - "book_author_link_book_fk": { - "ReferencedTable": "books", - "Columns": [ "book_id" ] - }, - "book_author_link_author_fk": { - "ReferencedTable": "authors", - "Columns": [ "author_id" ] - } - } }, "magazines": { - "ForeignKeys": { - }, "HttpVerbs": { "GET": { "AuthorizationType": "Anonymous" From 5a10b822990ab979b88c4082d6b9e828da6dac3a Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 12 Apr 2022 19:19:21 -0700 Subject: [PATCH 08/25] Get foreign key info in bulk --- .../Resolvers/BaseSqlQueryBuilder.cs | 25 ++- .../Resolvers/IQueryBuilder.cs | 17 ++- .../Resolvers/MySqlQueryBuilder.cs | 29 ++-- .../MetadataProviders/ISqlMetadataProvider.cs | 11 ++ .../MySqlMetadataProvider.cs | 33 ++-- .../SqlGraphQLFileMetadataProvider.cs | 3 + .../MetadataProviders/SqlMetadataProvider.cs | 144 +++++++++++------- 7 files changed, 173 insertions(+), 89 deletions(-) diff --git a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs index 55d3f3df1f..b08ff5a6b1 100644 --- a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs @@ -12,6 +12,9 @@ namespace Azure.DataGateway.Service.Resolvers /// public abstract class BaseSqlQueryBuilder { + public const string SCHEMA_NAME_PARAM = "schemaName"; + public const string TABLE_NAME_PARAM = "tableName"; + /// /// Adds database specific quotes to string identifier /// @@ -261,11 +264,19 @@ public string JoinPredicateStrings(params string?[] predicateStrings) } /// - public virtual string BuildForeignKeyQuery(string schemaName, string tableName) + public virtual string BuildForeignKeyInfoQuery(int numberOfParameters) { + string[] schemaNameParams = + CreateParams(kindOfParam: SCHEMA_NAME_PARAM, numberOfParameters); + + string[] tableNameParams = + CreateParams(kindOfParam: TABLE_NAME_PARAM, numberOfParameters); + string tableSchemaParamsForInClause = string.Join(", @", schemaNameParams); + string tableNameParamsForInClause = string.Join(", @", tableNameParams); return $"" + $"SELECT " + - $"ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))} " + + $"ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, " + + $"ReferencingColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, " + $"ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, " + $"ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, " + $"ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} " + @@ -282,9 +293,15 @@ public virtual string BuildForeignKeyQuery(string schemaName, string tableName) $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA " + $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME " + $"WHERE " + - $"ReferencingColumnUsage.SCHEMA_NAME = {QuoteIdentifier($"@{nameof(schemaName)}")} " + - $"AND ReferencingColumnUsage.TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + + $"ReferencingColumnUsage.TABLE_SCHEMA IN (@{tableSchemaParamsForInClause})" + + $"AND ReferencingColumnUsage.TABLE_NAME IN (@{tableNameParamsForInClause})" + $"AND ReferencingColumnUsage.ORDINAL_POSITION = ReferencedColumnUsage.ORDINAL_POSITION;"; } + + /// + public string[] CreateParams(string kindOfParam, int numberOfParameters) + { + return Enumerable.Range(0, numberOfParameters).Select(i => kindOfParam + i).ToArray(); + } } } diff --git a/DataGateway.Service/Resolvers/IQueryBuilder.cs b/DataGateway.Service/Resolvers/IQueryBuilder.cs index de27d6324d..47a74f2da2 100644 --- a/DataGateway.Service/Resolvers/IQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/IQueryBuilder.cs @@ -37,9 +37,20 @@ public interface IQueryBuilder public string Build(SqlUpsertQueryStructure structure); /// - /// Builds the query to obtain foreign key information for the given - /// schema, table name. + /// Builds the query to obtain foreign key information with the given + /// number of parameters. /// - public string BuildForeignKeyQuery(string schemaName, string tableName); + public string BuildForeignKeyInfoQuery(int numberOfParameters); + + /// + /// Creates a list of named parameters with incremental suffixes + /// starting from 0 to numberOfParameters - 1. + /// e.g. tableName0, tableName1 + /// + /// The kind of parameter being created acting + /// as the prefix common to all parameters. + /// The number of parameters to create. + /// The created list + public string[] CreateParams(string kindOfParam, int numberOfParameters); } } diff --git a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs index 261610a353..d2ff8eb904 100644 --- a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs @@ -14,8 +14,11 @@ namespace Azure.DataGateway.Service.Resolvers public class MySqlQueryBuilder : BaseSqlQueryBuilder, IQueryBuilder { private static DbCommandBuilder _builder = new MySqlCommandBuilder(); + public const string DATABASE_NAME_PARAM = "databaseName"; - /// + /// + /// Adds database specific quotes to string identifier + /// protected override string QuoteIdentifier(string ident) { return _builder.QuoteIdentifier(ident); @@ -146,24 +149,24 @@ public string Build(SqlUpsertQueryStructure structure) } /// - /// For MySql, the table name is only a 2 part name. - /// The schema name passed here is actually the database - /// from the connection string and that is compared against the SCHEMA_NAME - /// in the view KEY_COLUMN_USAGE. - /// - public override string BuildForeignKeyQuery(string databaseName, string tableName) + public override string BuildForeignKeyInfoQuery(int numberOfParameters) { + string[] databaseNameParams = CreateParams(DATABASE_NAME_PARAM, numberOfParameters); + string[] tableNameParams = CreateParams(TABLE_NAME_PARAM, numberOfParameters); + string tableSchemaParamsForInClause = string.Join(", ", databaseNameParams); + string tableNameParamsForInClause = string.Join(", ", tableNameParams); return $"" + $"SELECT " + - $"CONSTRAINT_NAME {QuoteIdentifier("Foreign Key Name")}" + - $"COLUMN_NAME {QuoteIdentifier("Referencing Column")}, " + - $"REFERENCED_TABLE_NAME {QuoteIdentifier("Referenced Table")}, " + - $"REFERENCED_COLUMN_NAME {QuoteIdentifier("Referenced Column")} " + + $"CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, " + + $"TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, " + + $"COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, " + + $"REFERENCED_TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, " + + $"REFERENCED_COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} " + $"FROM " + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + $"WHERE " + - $"SCHEMA_NAME = {QuoteIdentifier($"@{nameof(databaseName)}")} " + - $"AND TABLE_NAME = {QuoteIdentifier($"@{nameof(tableName)}")} " + + $"TABLE_SCHEMA = @{tableSchemaParamsForInClause} " + + $"AND TABLE_NAME = @{tableNameParamsForInClause} " + $"AND REFERENCED_TABLE_NAME IS NOT NULL " + $"AND REFERENCED_COLUMN_NAME IS NOT NULL;"; } diff --git a/DataGateway.Service/Services/MetadataProviders/ISqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/ISqlMetadataProvider.cs index 5a2496580e..13bcd6f685 100644 --- a/DataGateway.Service/Services/MetadataProviders/ISqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/ISqlMetadataProvider.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Data; using System.Threading.Tasks; using Azure.DataGateway.Service.Models; @@ -28,5 +29,15 @@ public Task PopulateTableDefinitionAsync( string schemaName, string tableName, TableDefinition tableDefinition); + + /// + /// Fills the table definition with information of the foreign keys + /// for all the tables. + /// + /// Name of the default schema. + /// Dictionary of all tables. + public Task PopulateForeignKeyDefinitionAsync( + string schemaName, + Dictionary tables); } } diff --git a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs index 9b8801ed38..57e0bc58ff 100644 --- a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs @@ -57,28 +57,33 @@ protected override async Task GetColumnsAsync( return allColumns; } - /// - /// For MySql, the table name is only a 2 part name. - /// The database name from the connection string needs to be used instead of schemaName. - /// - protected override string GetForeignKeyQuery(string schemaName, string tableName) - { - using MySqlConnection conn = new(ConnectionString); - return SqlQueryBuilder!.BuildForeignKeyQuery(conn.Database, tableName); - } - /// /// For MySql, the table name is only a 2 part name. /// The database name from the connection string needs to be used instead of schemaName. /// protected override Dictionary - GetForeignKeyQueryParams(string schemaName, string tableName) + GetForeignKeyQueryParams( + string[] schemaNames, + string[] tableNames) { using MySqlConnection conn = new(ConnectionString); - string databaseName = conn.Database; Dictionary parameters = new(); - parameters.Add(nameof(databaseName), databaseName); - parameters.Add(nameof(tableName), tableName); + + string[] databaseNameParams = + SqlQueryBuilder!.CreateParams( + kindOfParam: MySqlQueryBuilder.DATABASE_NAME_PARAM, + schemaNames.Count()); + string[] tableNameParams = + SqlQueryBuilder!.CreateParams( + kindOfParam: BaseSqlQueryBuilder.TABLE_NAME_PARAM, + tableNames.Count()); + + Enumerable.Range(0, schemaNames.Count()) + .Select(i => parameters.TryAdd(databaseNameParams[i], conn.Database)); + + Enumerable.Range(0, tableNames.Count()) + .Select(i => parameters.TryAdd(tableNameParams[i], tableNames[i])); + return parameters; } } diff --git a/DataGateway.Service/Services/MetadataProviders/SqlGraphQLFileMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlGraphQLFileMetadataProvider.cs index 7e4b75a213..06f21a7255 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlGraphQLFileMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlGraphQLFileMetadataProvider.cs @@ -106,6 +106,9 @@ private async Task EnrichDatabaseSchemaWithTableMetadata() $"is not supported."); } } + + await _sqlMetadataProvider!.PopulateForeignKeyDefinitionAsync(schemaName, GraphQLResolverConfig.DatabaseSchema.Tables); + } private void InitFilterParser() diff --git a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs index 17803afac8..41d184abaa 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading.Tasks; using Azure.DataGateway.Service.Configurations; +using Azure.DataGateway.Service.Exceptions; using Azure.DataGateway.Service.Models; using Azure.DataGateway.Service.Resolvers; using Microsoft.Extensions.Options; @@ -88,11 +89,71 @@ public virtual async Task PopulateTableDefinitionAsync( PopulateColumnDefinitionWithHasDefault( tableDefinition, columnsInTable); + } + + /// + public async Task PopulateForeignKeyDefinitionAsync( + string defaultSchemaName, + Dictionary tables) + { + // Build the query required to get the foreign key information. + string queryForForeignKeyInfo = + ((BaseSqlQueryBuilder)SqlQueryBuilder!).BuildForeignKeyInfoQuery(tables.Count); + + // Build the array storing all the schemaNames, for now the defaultSchemaName. + string[] schemaNames = Enumerable.Range(1, tables.Count).Select(x => defaultSchemaName).ToArray(); + + // Build the parameters for dictionary for the foreign key info query + // consisting of all schema names and table names. + Dictionary parameters = + GetForeignKeyQueryParams(schemaNames, tables.Keys.ToArray()); + + // Execute the foreign key info query. + using DbDataReader reader = + await _queryExecutor!.ExecuteQueryAsync(queryForForeignKeyInfo, parameters); + + // Extract the first row from the result. + Dictionary? foreignKeyInfo = + await _queryExecutor!.ExtractRowFromDbDataReader(reader); - await PopulateForeignKeyDefinitionAsync( - schemaName, - tableName, - tableDefinition); + // While the result is not null + // keep populating the table definition for all tables with all foreign keys. + while (foreignKeyInfo != null) + { + string twoPartTableName = (string)foreignKeyInfo[nameof(TableDefinition)]!; + TableDefinition? tableDefinition; + string foreignKeyName = (string)foreignKeyInfo[nameof(ForeignKeyDefinition)]!; + ForeignKeyDefinition? foreignKeyDefinition; + + if (tables.TryGetValue(twoPartTableName, out tableDefinition)) + { + if (!tableDefinition.ForeignKeys.TryGetValue(foreignKeyName, out foreignKeyDefinition)) + { + // If this is the first column in this foreign key for this table, + // add the referenced table to the tableDefinition. + foreignKeyDefinition = new(); + foreignKeyDefinition.ReferencedTable = + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedTable)]!; + tableDefinition.ForeignKeys.Add(foreignKeyName, foreignKeyDefinition); + } + + // add the referenced and referencing columns to the foreign key definition. + foreignKeyDefinition.ReferencedColumns.Add( + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedColumns)]!); + foreignKeyDefinition.ReferencingColumns.Add( + (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencingColumns)]!); + } + else + { + // This should not happen. + throw new DataGatewayException( + message: "Foreign key information is retrieved for a table that is not to be exposed.", + statusCode: System.Net.HttpStatusCode.InternalServerError, + subStatusCode: DataGatewayException.SubStatusCodes.UnexpectedError); + } + + foreignKeyInfo = await _queryExecutor.ExtractRowFromDbDataReader(reader); + } } /// @@ -193,65 +254,38 @@ protected void PopulateColumnDefinitionWithHasDefault( } } - protected async Task PopulateForeignKeyDefinitionAsync( - string schemaName, - string tableName, - TableDefinition tableDefinition) - { - string queryForForeignKeyInfo = GetForeignKeyQuery(schemaName, tableName); - Dictionary parameters = - GetForeignKeyQueryParams(schemaName, tableName); - using DbDataReader reader = - await _queryExecutor!.ExecuteQueryAsync(queryForForeignKeyInfo, parameters); - - Dictionary? foreignKeyInfo = - await _queryExecutor!.ExtractRowFromDbDataReader(reader); - - while (foreignKeyInfo != null) - { - string foreignKeyName = (string)foreignKeyInfo[nameof(ForeignKeyDefinition)]!; - ForeignKeyDefinition? foreignKeyDefinition; - if (!tableDefinition.ForeignKeys.TryGetValue(foreignKeyName, out foreignKeyDefinition)) - { - foreignKeyDefinition = new(); - foreignKeyDefinition.ReferencedTable = - (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedTable)]!; - } - - foreignKeyDefinition.ReferencedColumns.Add( - (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencedColumns)]!); - foreignKeyDefinition.ReferencingColumns.Add( - (string)foreignKeyInfo[nameof(ForeignKeyDefinition.ReferencingColumns)]!); - - foreignKeyInfo = await _queryExecutor.ExtractRowFromDbDataReader(reader); - } - } - - /// - /// Invokes the underlying query builder to - /// build the query useful for retrieving the foreign key information. - /// - /// - /// - /// The Sql query to use. - protected virtual string GetForeignKeyQuery(string schemaName, string tableName) - { - return SqlQueryBuilder!.BuildForeignKeyQuery(schemaName, tableName); - } - /// /// Builds the dictionary of parameters and their values required for the /// foreign key query. /// - /// - /// + /// + /// /// The dictionary populated with parameters. protected virtual Dictionary - GetForeignKeyQueryParams(string schemaName, string tableName) + GetForeignKeyQueryParams( + string[] schemaNames, + string[] tableNames) { Dictionary parameters = new(); - parameters.Add(nameof(schemaName), schemaName); - parameters.Add(nameof(tableName), tableName); + string[] schemaNameParams = + SqlQueryBuilder!.CreateParams( + kindOfParam: BaseSqlQueryBuilder.SCHEMA_NAME_PARAM, + schemaNames.Count()); + string[] tableNameParams = + SqlQueryBuilder!.CreateParams( + kindOfParam: BaseSqlQueryBuilder.TABLE_NAME_PARAM, + tableNames.Count()); + + for (int i = 0; i < schemaNames.Count(); ++i) + { + parameters.Add(schemaNameParams[i], schemaNames[i]); + } + + for (int i = 0; i < tableNames.Count(); ++i) + { + parameters.Add(tableNameParams[i], tableNames[i]); + } + return parameters; } } From 3f3e008268d69c4b95398ac945bc72491866a02c Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 12 Apr 2022 19:20:02 -0700 Subject: [PATCH 09/25] Fix tests --- DataGateway.Service/MsSqlBooks.sql | 8 +++++--- DataGateway.Service/MySqlBooks.sql | 3 ++- DataGateway.Service/PostgreSqlBooks.sql | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/DataGateway.Service/MsSqlBooks.sql b/DataGateway.Service/MsSqlBooks.sql index 1340309492..5ac58ff4df 100644 --- a/DataGateway.Service/MsSqlBooks.sql +++ b/DataGateway.Service/MsSqlBooks.sql @@ -5,9 +5,9 @@ DROP TABLE IF EXISTS book_website_placements; DROP TABLE IF EXISTS books; DROP TABLE IF EXISTS publishers; DROP TABLE IF EXISTS magazines; -DROP TABLE IF EXISTS comics; -DROP TABLE IF EXISTS stocks; DROP TABLE IF EXISTS stocks_price; +DROP TABLE IF EXISTS stocks; +DROP TABLE IF EXISTS comics; --Autogenerated id seed are set at 5001 for consistency with Postgres --This allows for tests using the same id values for both languages @@ -143,4 +143,6 @@ SET IDENTITY_INSERT reviews ON INSERT INTO reviews(id, book_id, content) VALUES (567, 1, 'Indeed a great book'), (568, 1, 'I loved it'), (569, 1, 'best book I read in years'); SET IDENTITY_INSERT reviews OFF -INSERT INTO stocks(categoryid, pieceid, categoryName) VALUES (1, 1, 'books'), (2, 1, 'magazines'); +INSERT INTO comics(id, title, categoryName) VALUES (1, 'Star Trek', 'SciFi'), (2, 'Cinderella', 'FairyTales'); + +INSERT INTO stocks(categoryid, pieceid, categoryName) VALUES (1, 1, 'SciFi'), (2, 1, 'FairyTales'); diff --git a/DataGateway.Service/MySqlBooks.sql b/DataGateway.Service/MySqlBooks.sql index 52bf3d9707..0bb6693975 100644 --- a/DataGateway.Service/MySqlBooks.sql +++ b/DataGateway.Service/MySqlBooks.sql @@ -124,7 +124,8 @@ INSERT INTO books(id, title, publisher_id) VALUES (1, 'Awesome book', 1234), (2, INSERT INTO book_website_placements(book_id, price) VALUES (1, 100), (2, 50), (3, 23), (5, 33); INSERT INTO book_author_link(book_id, author_id) VALUES (1, 123), (2, 124), (3, 123), (3, 124), (4, 123), (4, 124); INSERT INTO reviews(id, book_id, content) VALUES (567, 1, 'Indeed a great book'), (568, 1, 'I loved it'), (569, 1, 'best book I read in years'); -INSERT INTO stocks(categoryid, pieceid,categoryName) VALUES (1, 1, 'books'), (2, 1, 'magazines'); +INSERT INTO comics(id, title, categoryName) VALUES (1, 'Star Trek', 'SciFi'), (2, 'Cinderella', 'FairyTales'); +INSERT INTO stocks(categoryid, pieceid, categoryName) VALUES (1, 1, 'SciFi'), (2, 1, 'FairyTales'); -- Starting with id > 5000 is chosen arbitrarily so that the incremented id-s won't conflict with the manually inserted ids in this script -- AUTO_INCREMENT is set to 5001 so the next autogenerated id will be 5001 diff --git a/DataGateway.Service/PostgreSqlBooks.sql b/DataGateway.Service/PostgreSqlBooks.sql index 7bd2a3b19e..8d7693b284 100644 --- a/DataGateway.Service/PostgreSqlBooks.sql +++ b/DataGateway.Service/PostgreSqlBooks.sql @@ -123,8 +123,8 @@ INSERT INTO books(id, title, publisher_id) VALUES (1, 'Awesome book', 1234), (2, INSERT INTO book_website_placements(book_id, price) VALUES (1, 100), (2, 50), (3, 23), (5, 33); INSERT INTO book_author_link(book_id, author_id) VALUES (1, 123), (2, 124), (3, 123), (3, 124), (4, 123), (4, 124); INSERT INTO reviews(id, book_id, content) VALUES (567, 1, 'Indeed a great book'), (568, 1, 'I loved it'), (569, 1, 'best book I read in years'); -INSERT INTO stocks(categoryid, pieceid,categoryName) VALUES (1, 1, 'books'), (2, 1, 'magazines'); - +INSERT INTO comics(id, title, categoryName) VALUES (1, 'Star Trek', 'SciFi'), (2, 'Cinderella', 'FairyTales'); +INSERT INTO stocks(categoryid, pieceid, categoryName) VALUES (1, 1, 'SciFi'), (2, 1, 'FairyTales'); --Starting with id > 5000 is chosen arbitrarily so that the incremented id-s won't conflict with the manually inserted ids in this script --Sequence counter is set to 5000 so the next autogenerated id will be 5001 From 6293125549837d3dbe80b1f483fea49c1488ba1c Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 12 Apr 2022 19:20:24 -0700 Subject: [PATCH 10/25] Fix stocks and comics test values to conform constraint --- .../SqlTests/MsSqlRestApiTests.cs | 11 ++++++----- DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs b/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs index 09344e0153..afef22b054 100644 --- a/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs +++ b/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs @@ -190,7 +190,7 @@ public class MsSqlRestApiTests : RestApiTestBase // the insertion. $"SELECT [categoryid],[pieceid],[categoryName],[piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 5 AND [pieceid] = 2 AND [categoryName] = 'Thriller' " + + $"WHERE [categoryid] = 5 AND [pieceid] = 2 AND [categoryName] = 'FairyTales' " + $"AND [piecesAvailable] = 0 AND [piecesRequired] = 0 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -267,6 +267,7 @@ public class MsSqlRestApiTests : RestApiTestBase "PutOne_Insert_AutoGenNonPK_Test", $"SELECT [id], [title], [volume] FROM { _integration_AutoGenNonPK_TableName } " + $"WHERE id = { STARTING_ID_FOR_TEST_INSERTS } AND [title] = 'Star Trek' " + + $"AND [categoryName] = 'SciFi' " + $"AND [volume] IS NOT NULL " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -274,7 +275,7 @@ public class MsSqlRestApiTests : RestApiTestBase "PutOne_Insert_CompositeNonAutoGenPK_Test", $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 3 AND [pieceid] = 1 AND [categoryName] = 'comics' " + + $"WHERE [categoryid] = 3 AND [pieceid] = 1 AND [categoryName] = 'SciFi' " + $"AND [piecesAvailable] = 2 AND [piecesRequired] = 1 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -297,7 +298,7 @@ public class MsSqlRestApiTests : RestApiTestBase "PatchOne_Insert_CompositeNonAutoGenPK_Test", $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 4 AND [pieceid] = 1 AND [categoryName] = 'Suspense' " + + $"WHERE [categoryid] = 4 AND [pieceid] = 1 AND [categoryName] = 'FairyTales' " + $"AND [piecesAvailable] = 5 AND [piecesRequired] = 4 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -305,7 +306,7 @@ public class MsSqlRestApiTests : RestApiTestBase "PatchOne_Insert_Default_Test", $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 7 AND [pieceid] = 1 AND [categoryName] = 'Drama' " + + $"WHERE [categoryid] = 7 AND [pieceid] = 1 AND [categoryName] = 'SciFi' " + $"AND [piecesAvailable] = 0 AND [piecesRequired] = 0 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -332,7 +333,7 @@ public class MsSqlRestApiTests : RestApiTestBase "PatchOne_Update_CompositeNonAutoGenPK_Test", $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 1 AND [pieceid] = 1 AND [categoryName] = 'books' " + + $"WHERE [categoryid] = 1 AND [pieceid] = 1 AND [categoryName] = 'SciFi' " + $"AND [piecesAvailable]= 10 AND [piecesRequired] = 0 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, diff --git a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs index e9cf430edd..6a5873c7f4 100644 --- a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs +++ b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs @@ -391,7 +391,7 @@ await SetupAndRunRestApiTest( { ""categoryid"": ""5"", ""pieceid"": ""2"", - ""categoryName"":""Thriller"" + ""categoryName"":""FairyTab"" }"; expectedLocationHeader = $"categoryid/5/pieceid/2"; @@ -633,7 +633,8 @@ await SetupAndRunRestApiTest( // that is autogenerated. requestBody = @" { - ""title"": ""Star Trek"" + ""title"": ""Star Trek"", + ""categoryName"" : ""SciFi"" }"; expectedLocationHeader = $"id/{STARTING_ID_FOR_TEST_INSERTS}"; @@ -651,7 +652,7 @@ await SetupAndRunRestApiTest( requestBody = @" { - ""categoryName"":""comics"", + ""categoryName"":""SciFi"", ""piecesAvailable"":""2"", ""piecesRequired"":""1"" }"; @@ -719,7 +720,7 @@ await SetupAndRunRestApiTest( requestBody = @" { - ""categoryName"": ""Suspense"", + ""categoryName"": ""FairyTales"", ""piecesAvailable"":""5"", ""piecesRequired"":""4"" }"; @@ -739,7 +740,7 @@ await SetupAndRunRestApiTest( requestBody = @" { - ""categoryName"": ""Drama"" + ""categoryName"": ""SciFi"" }"; expectedLocationHeader = $"categoryid/7/pieceid/1"; From c0e151cc08f7cb6e2895aec146f7445c484fd0f2 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 12 Apr 2022 19:20:45 -0700 Subject: [PATCH 11/25] Add book_website_placement to sql-config-test --- .../sql-config-test.json | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/DataGateway.Service.Tests/sql-config-test.json b/DataGateway.Service.Tests/sql-config-test.json index 3cd004355a..22eaee6f69 100644 --- a/DataGateway.Service.Tests/sql-config-test.json +++ b/DataGateway.Service.Tests/sql-config-test.json @@ -174,6 +174,32 @@ } } }, + "book_website_placements": { + "PrimaryKey": [ "id" ], + "ForeignKeys": { + "book_website_placement_book_fk": { + "ReferencedTable": "books", + "Columns": [ "book_id" ] + } + }, + "HttpVerbs": { + "GET": { + "AuthorizationType": "Anonymous" + }, + "POST": { + "AuthorizationType": "Authenticated" + }, + "DELETE": { + "Authorization": "Authenticated" + }, + "PUT": { + "Authorization": "Authenticated" + }, + "PATCH": { + "Authorization": "Authenticated" + } + } + }, "authors": { "PrimaryKey": [ "id" ], "Columns": { From b065be99b1f00c142ed1257b4f516223222d4703 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 12 Apr 2022 19:25:43 -0700 Subject: [PATCH 12/25] Fix Put update test --- DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs | 8 ++++---- DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs b/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs index afef22b054..447f80d38c 100644 --- a/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs +++ b/DataGateway.Service.Tests/SqlTests/MsSqlRestApiTests.cs @@ -237,9 +237,9 @@ public class MsSqlRestApiTests : RestApiTestBase }, { "PutOne_Update_CompositeNonAutoGenPK_Test", - $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," + + $"SELECT [categoryid], [pieceid], [categoryName], [piecesAvailable]," + $"[piecesRequired] FROM { _Composite_NonAutoGenPK } " + - $"WHERE [categoryid] = 2 AND [pieceid] = 1 AND [categoryName] = 'History' " + + $"WHERE [categoryid] = 2 AND [pieceid] = 1 AND [categoryName] = 'SciFi' " + $"AND [piecesAvailable] = 10 AND [piecesRequired] = 5 " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, @@ -265,9 +265,9 @@ public class MsSqlRestApiTests : RestApiTestBase }, { "PutOne_Insert_AutoGenNonPK_Test", - $"SELECT [id], [title], [volume] FROM { _integration_AutoGenNonPK_TableName } " + + $"SELECT [id], [title], [volume], [categoryName] FROM { _integration_AutoGenNonPK_TableName } " + $"WHERE id = { STARTING_ID_FOR_TEST_INSERTS } AND [title] = 'Star Trek' " + - $"AND [categoryName] = 'SciFi' " + + $"AND [categoryName] = 'Suspense' " + $"AND [volume] IS NOT NULL " + $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER" }, diff --git a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs index 6a5873c7f4..3051adbbc0 100644 --- a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs +++ b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs @@ -521,7 +521,7 @@ await SetupAndRunRestApiTest( requestBody = @" { - ""categoryName"":""History"", + ""categoryName"":""SciFi"", ""piecesAvailable"":""10"", ""piecesRequired"":""5"" }"; @@ -634,7 +634,7 @@ await SetupAndRunRestApiTest( requestBody = @" { ""title"": ""Star Trek"", - ""categoryName"" : ""SciFi"" + ""categoryName"" : ""Suspense"" }"; expectedLocationHeader = $"id/{STARTING_ID_FOR_TEST_INSERTS}"; From 2206f51d82e43041c998a94e91141958a4ab6a94 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 00:12:58 -0700 Subject: [PATCH 13/25] Fix Rest API tests for categoryName constraint --- .../SqlTests/MySqlRestApiTests.cs | 16 ++++++++-------- .../SqlTests/PostgreSqlRestApiTests.cs | 14 +++++++------- .../SqlTests/RestApiTestBase.cs | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/MySqlRestApiTests.cs b/DataGateway.Service.Tests/SqlTests/MySqlRestApiTests.cs index 7bffa35af0..46e0a3b771 100644 --- a/DataGateway.Service.Tests/SqlTests/MySqlRestApiTests.cs +++ b/DataGateway.Service.Tests/SqlTests/MySqlRestApiTests.cs @@ -353,7 +353,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 5 AND pieceid = 2 AND categoryName ='Thriller' AND piecesAvailable = 0 + WHERE categoryid = 5 AND pieceid = 2 AND categoryName ='FairyTales' AND piecesAvailable = 0 AND piecesRequired = 0 ) AS subq " @@ -436,7 +436,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 2 AND pieceid = 1 AND categoryName ='History' AND piecesAvailable = 10 + WHERE categoryid = 2 AND pieceid = 1 AND categoryName ='SciFi' AND piecesAvailable = 10 AND piecesRequired = 5 ) AS subq " @@ -466,9 +466,9 @@ AND issueNumber is NULL }, { "PutOne_Insert_AutoGenNonPK_Test", - @"SELECT JSON_OBJECT('id', id, 'title', title, 'volume', volume ) AS data + @"SELECT JSON_OBJECT('id', id, 'title', title, 'volume', volume, 'categoryName', categoryName) AS data FROM ( - SELECT id, title, volume + SELECT id, title, volume, categoryName FROM " + _integration_AutoGenNonPK_TableName + @" WHERE id = " + $"{STARTING_ID_FOR_TEST_INSERTS}" + @" AND title = 'Star Trek' AND volume IS NOT NULL @@ -483,7 +483,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 3 AND pieceid = 1 AND categoryName ='comics' AND piecesAvailable = 2 + WHERE categoryid = 3 AND pieceid = 1 AND categoryName ='SciFi' AND piecesAvailable = 2 AND piecesRequired = 1 ) AS subq " @@ -520,7 +520,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 4 AND pieceid = 1 AND categoryName ='Suspense' AND piecesAvailable = 5 + WHERE categoryid = 4 AND pieceid = 1 AND categoryName ='FairyTales' AND piecesAvailable = 5 AND piecesRequired = 4 ) AS subq " @@ -533,7 +533,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 7 AND pieceid = 1 AND categoryName ='Drama' AND piecesAvailable = 0 + WHERE categoryid = 7 AND pieceid = 1 AND categoryName ='SciFi' AND piecesAvailable = 0 AND piecesRequired = 0 ) AS subq " @@ -581,7 +581,7 @@ SELECT JSON_OBJECT('categoryid', categoryid, 'pieceid', pieceid, 'categoryName', FROM ( SELECT categoryid, pieceid, categoryName,piecesAvailable,piecesRequired FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 1 AND pieceid = 1 AND categoryName ='books' AND piecesAvailable = 10 + WHERE categoryid = 1 AND pieceid = 1 AND categoryName ='SciFi' AND piecesAvailable = 10 AND piecesRequired = 0 ) AS subq " diff --git a/DataGateway.Service.Tests/SqlTests/PostgreSqlRestApiTests.cs b/DataGateway.Service.Tests/SqlTests/PostgreSqlRestApiTests.cs index 70686a274f..00b5a0cc09 100644 --- a/DataGateway.Service.Tests/SqlTests/PostgreSqlRestApiTests.cs +++ b/DataGateway.Service.Tests/SqlTests/PostgreSqlRestApiTests.cs @@ -336,7 +336,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 5 AND pieceid = 2 AND ""categoryName"" = 'Thriller' + WHERE categoryid = 5 AND pieceid = 2 AND ""categoryName"" = 'FairyTales' AND ""piecesAvailable"" = 0 AND ""piecesRequired"" = 0 ) AS subq " @@ -417,7 +417,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 2 AND pieceid = 1 AND ""categoryName"" = 'History' + WHERE categoryid = 2 AND pieceid = 1 AND ""categoryName"" = 'SciFi' AND ""piecesAvailable"" = 10 AND ""piecesRequired"" = 5 ) AS subq " @@ -457,7 +457,7 @@ SELECT to_jsonb(subq) AS data @" SELECT to_jsonb(subq) AS data FROM ( - SELECT id, title, volume + SELECT id, title, volume, ""categoryName"" FROM " + _integration_AutoGenNonPK_TableName + @" WHERE id = " + STARTING_ID_FOR_TEST_INSERTS + @" AND title = 'Star Trek' AND volume IS NOT NULL @@ -471,7 +471,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 3 AND pieceid = 1 AND ""categoryName"" = 'comics' + WHERE categoryid = 3 AND pieceid = 1 AND ""categoryName"" = 'SciFi' AND ""piecesAvailable"" = 2 AND ""piecesRequired"" = 1 ) AS subq " @@ -506,7 +506,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 4 AND pieceid = 1 AND ""categoryName"" = 'Suspense' + WHERE categoryid = 4 AND pieceid = 1 AND ""categoryName"" = 'FairyTales' AND ""piecesAvailable"" = 5 AND ""piecesRequired"" = 4 ) AS subq " @@ -518,7 +518,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 7 AND pieceid = 1 AND ""categoryName"" = 'Drama' + WHERE categoryid = 7 AND pieceid = 1 AND ""categoryName"" = 'SciFi' AND ""piecesAvailable"" = 0 AND ""piecesRequired"" = 0 ) AS subq " @@ -563,7 +563,7 @@ SELECT to_jsonb(subq) AS data FROM ( SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired"" FROM " + _Composite_NonAutoGenPK + @" - WHERE categoryid = 1 AND pieceid = 1 AND ""categoryName"" = 'books' + WHERE categoryid = 1 AND pieceid = 1 AND ""categoryName"" = 'SciFi' AND ""piecesAvailable"" = 10 AND ""piecesRequired"" = 0 ) AS subq " diff --git a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs index 89c8d1d25f..3e8ee99ec7 100644 --- a/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs +++ b/DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs @@ -391,7 +391,7 @@ await SetupAndRunRestApiTest( { ""categoryid"": ""5"", ""pieceid"": ""2"", - ""categoryName"":""FairyTab"" + ""categoryName"":""FairyTales"" }"; expectedLocationHeader = $"categoryid/5/pieceid/2"; From fcf7c24aa4d1ffb5e11495b9124bfd808865f911 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 00:18:04 -0700 Subject: [PATCH 14/25] Enclose columnName in quotes for PostgreSql --- DataGateway.Service/PostgreSqlBooks.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DataGateway.Service/PostgreSqlBooks.sql b/DataGateway.Service/PostgreSqlBooks.sql index 600fb56bc3..8ef5a02b7f 100644 --- a/DataGateway.Service/PostgreSqlBooks.sql +++ b/DataGateway.Service/PostgreSqlBooks.sql @@ -55,7 +55,7 @@ CREATE TABLE comics( id bigint PRIMARY KEY, title text NOT NULL, volume bigint GENERATED BY DEFAULT AS IDENTITY, - categoryName varchar(100) NOT NULL UNIQUE + "categoryName" varchar(100) NOT NULL UNIQUE ); CREATE TABLE stocks( @@ -107,8 +107,8 @@ ON DELETE CASCADE; ALTER TABLE stocks ADD CONSTRAINT stocks_comics_fk -FOREIGN KEY (categoryName) -REFERENCES comics (categoryName) +FOREIGN KEY ("categoryName") +REFERENCES comics ("categoryName") ON DELETE CASCADE; ALTER TABLE stocks_price From 7285270731c0d3a9f8e5cb4f33a6d908498a9c98 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 00:22:08 -0700 Subject: [PATCH 15/25] Fix MySQL foreign query --- DataGateway.Service/Resolvers/MySqlQueryBuilder.cs | 8 ++++---- .../MetadataProviders/MySqlMetadataProvider.cs | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs index dc428e263e..e337b5636e 100644 --- a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs @@ -126,8 +126,8 @@ public override string BuildForeignKeyInfoQuery(int numberOfParameters) { string[] databaseNameParams = CreateParams(DATABASE_NAME_PARAM, numberOfParameters); string[] tableNameParams = CreateParams(TABLE_NAME_PARAM, numberOfParameters); - string tableSchemaParamsForInClause = string.Join(", ", databaseNameParams); - string tableNameParamsForInClause = string.Join(", ", tableNameParams); + string tableSchemaParamsForInClause = string.Join(", @", databaseNameParams); + string tableNameParamsForInClause = string.Join(", @", tableNameParams); return $"" + $"SELECT " + $"CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, " + @@ -138,8 +138,8 @@ public override string BuildForeignKeyInfoQuery(int numberOfParameters) $"FROM " + $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + $"WHERE " + - $"TABLE_SCHEMA = @{tableSchemaParamsForInClause} " + - $"AND TABLE_NAME = @{tableNameParamsForInClause} " + + $"TABLE_SCHEMA IN (@{tableSchemaParamsForInClause}) " + + $"AND TABLE_NAME IN (@{tableNameParamsForInClause}) " + $"AND REFERENCED_TABLE_NAME IS NOT NULL " + $"AND REFERENCED_COLUMN_NAME IS NOT NULL;"; } diff --git a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs index 57e0bc58ff..162c281613 100644 --- a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs @@ -78,11 +78,15 @@ protected override async Task GetColumnsAsync( kindOfParam: BaseSqlQueryBuilder.TABLE_NAME_PARAM, tableNames.Count()); - Enumerable.Range(0, schemaNames.Count()) - .Select(i => parameters.TryAdd(databaseNameParams[i], conn.Database)); + for (int i = 0; i < schemaNames.Count(); ++i) + { + parameters.Add(databaseNameParams[i], conn.Database); + } - Enumerable.Range(0, tableNames.Count()) - .Select(i => parameters.TryAdd(tableNameParams[i], tableNames[i])); + for (int i = 0; i < tableNames.Count(); ++i) + { + parameters.Add(tableNameParams[i], tableNames[i]); + } return parameters; } From de97e347bfb39542db9f9b7a48aed27e61c437ad Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 01:04:16 -0700 Subject: [PATCH 16/25] Add new foreign key tests --- .../SqlTests/SqlMetadataProviderTests.cs | 5 ++ .../sql-config-test.json | 80 +++++++++++++++++-- .../Services/EdmModelBuilder.cs | 28 ++++--- DataGateway.Service/sql-config.json | 2 + 4 files changed, 99 insertions(+), 16 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs b/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs index b8ad107b93..bedd539b3e 100644 --- a/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs +++ b/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs @@ -31,6 +31,11 @@ SqlGraphQLFileMetadataProvider expectedMetadataProvider actualTableDefinition.PrimaryKey, $"Did not find the expected primary keys for table {tableName}"); + CollectionAssert.AreEqual( + expectedTableDefinition.ForeignKeys, + actualTableDefinition.ForeignKeys, + $"Did not find the expected foreign keys for table {tableName}"); + foreach ((string columnName, ColumnDefinition expectedColumnDefinition) in expectedTableDefinition.Columns) { ColumnDefinition actualColumnDefinition; diff --git a/DataGateway.Service.Tests/sql-config-test.json b/DataGateway.Service.Tests/sql-config-test.json index 22eaee6f69..2d4fd4f5ec 100644 --- a/DataGateway.Service.Tests/sql-config-test.json +++ b/DataGateway.Service.Tests/sql-config-test.json @@ -153,7 +153,7 @@ "ForeignKeys": { "book_publisher_fk": { "ReferencedTable": "publishers", - "Columns": [ "publisher_id" ] + "ReferencingColumns": [ "publisher_id" ] } }, "HttpVerbs": { @@ -179,7 +179,7 @@ "ForeignKeys": { "book_website_placement_book_fk": { "ReferencedTable": "books", - "Columns": [ "book_id" ] + "ReferencingColumns": [ "book_id" ] } }, "HttpVerbs": { @@ -238,7 +238,7 @@ "ForeignKeys": { "review_book_fk": { "ReferencedTable": "books", - "Columns": [ "book_id" ] + "ReferencingColumns": [ "book_id" ] } }, "HttpVerbs": { @@ -260,11 +260,11 @@ "ForeignKeys": { "book_author_link_book_fk": { "ReferencedTable": "books", - "Columns": [ "book_id" ] + "ReferencingColumns": [ "book_id" ] }, "book_author_link_author_fk": { "ReferencedTable": "authors", - "Columns": [ "author_id" ] + "ReferencingColumns": [ "author_id" ] } } }, @@ -321,6 +321,10 @@ "Type": "bigint", "IsAutoGenerated": true, "IsNullable": false + }, + "categoryName": { + "Type": "text", + "IsNullable": false } }, "HttpVerbs": { @@ -337,6 +341,72 @@ "Authorization": "Authenticated" } } + }, + "stocks": { + "PrimaryKey": [ "categoryid, pieceid" ], + "Columns": { + "categoryid": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": false + }, + "pieceid": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": false + }, + "categoryName": { + "Type": "text", + "IsNullable": false + }, + "piecesAvailable": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": true, + "HasDefault": true + }, + "piecesRequired": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": false, + "HasDefault": true + } + }, + "ForeignKeys": { + "stocks_comics_fk": { + "ReferencedTable": "comics", + "ReferencedColumns": [ "categoryName" ], + "ReferencingColumns": [ "categoryName" ] + } + } + }, + "stocks_price": { + "PrimaryKey": [ "categoryid, pieceid", "instant" ], + "Columns": { + "categoryid": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": false + }, + "pieceid": { + "Type": "bigint", + "IsAutoGenerated": false, + "IsNullable": false + }, + "instant": { + "Type": "timestamp" + }, + "price": { + "Type": "float" + } + }, + "ForeignKeys": { + "stocks_price_stocks_fk": { + "ReferencedTable": "stocks", + "ReferencedColumns": [ "categoryid", "pieceid" ], + "ReferencingColumns": [ "categoryid", "pieceid" ] + } + } } } } diff --git a/DataGateway.Service/Services/EdmModelBuilder.cs b/DataGateway.Service/Services/EdmModelBuilder.cs index 9f757e74bb..3cf3e57119 100644 --- a/DataGateway.Service/Services/EdmModelBuilder.cs +++ b/DataGateway.Service/Services/EdmModelBuilder.cs @@ -51,18 +51,24 @@ private EdmModelBuilder BuildEntityTypes(DatabaseSchema schema) // need to convert our column system type to an Edm type Type columnSystemType = schema.Tables[entityName].Columns[column].SystemType; EdmPrimitiveTypeKind type = EdmPrimitiveTypeKind.None; - if (ReferenceEquals(typeof(string), columnSystemType)) - { - type = EdmPrimitiveTypeKind.String; - } - else if (ReferenceEquals(typeof(long), columnSystemType)) - { - type = EdmPrimitiveTypeKind.Int64; - } - else + + switch(columnSystemType) { - throw new ArgumentException($"No resolver for column type" + - $" {columnSystemType.Name}"); + case string: + type = EdmPrimitiveTypeKind.String; + break; + case TypeCode.Int64: + type = EdmPrimitiveTypeKind.Int64; + break; + case TypeCode.Byte: + type = EdmPrimitiveTypeKind.Byte; + break; + case Type.GetTypeCode(byte[]): + type = EdmPrimitiveTypeKind.TimeOfDay; + break; + default: + throw new ArgumentException($"No resolver for column type" + + $" {columnSystemType.Name}"); } // if column is in our list of keys we add as a key to entity diff --git a/DataGateway.Service/sql-config.json b/DataGateway.Service/sql-config.json index 295bbe918a..dead79f08b 100644 --- a/DataGateway.Service/sql-config.json +++ b/DataGateway.Service/sql-config.json @@ -245,6 +245,8 @@ "Authorization": "Authenticated" } } + }, + "stocks_price": { } } } From 5a11193f64150dd913bf35884c6427de39c88ec7 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 11:53:41 -0700 Subject: [PATCH 17/25] Modify sql-config-test to explicitly include referenced columns in foreign keys --- .../Azure.DataGateway.Service.Tests.csproj | 2 +- DataGateway.Service.Tests/sql-config-test.json | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj b/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj index cb940cf911..4bb00a563e 100644 --- a/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj +++ b/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj @@ -28,7 +28,7 @@ - PreserveNewest + Always diff --git a/DataGateway.Service.Tests/sql-config-test.json b/DataGateway.Service.Tests/sql-config-test.json index 2d4fd4f5ec..8f7f87aaca 100644 --- a/DataGateway.Service.Tests/sql-config-test.json +++ b/DataGateway.Service.Tests/sql-config-test.json @@ -153,6 +153,7 @@ "ForeignKeys": { "book_publisher_fk": { "ReferencedTable": "publishers", + "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "publisher_id" ] } }, @@ -179,6 +180,7 @@ "ForeignKeys": { "book_website_placement_book_fk": { "ReferencedTable": "books", + "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "book_id" ] } }, @@ -238,6 +240,7 @@ "ForeignKeys": { "review_book_fk": { "ReferencedTable": "books", + "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "book_id" ] } }, @@ -260,10 +263,12 @@ "ForeignKeys": { "book_author_link_book_fk": { "ReferencedTable": "books", + "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "book_id" ] }, "book_author_link_author_fk": { "ReferencedTable": "authors", + "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "author_id" ] } } @@ -285,8 +290,6 @@ "IsNullable": true } }, - "ForeignKeys": { - }, "HttpVerbs": { "GET": { "AuthorizationType": "Anonymous" @@ -343,7 +346,7 @@ } }, "stocks": { - "PrimaryKey": [ "categoryid, pieceid" ], + "PrimaryKey": [ "categoryid", "pieceid" ], "Columns": { "categoryid": { "Type": "bigint", @@ -381,7 +384,7 @@ } }, "stocks_price": { - "PrimaryKey": [ "categoryid, pieceid", "instant" ], + "PrimaryKey": [ "categoryid", "pieceid", "instant" ], "Columns": { "categoryid": { "Type": "bigint", @@ -397,7 +400,8 @@ "Type": "timestamp" }, "price": { - "Type": "float" + "Type": "float", + "IsNullable": true } }, "ForeignKeys": { From 685c700439e1fa8793b023b8d5eb22af364eaac8 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 11:54:09 -0700 Subject: [PATCH 18/25] Compare ForeignKeyDefinitions --- .../SqlTests/SqlMetadataProviderTests.cs | 7 +++++-- DataGateway.Service/Models/DatabaseSchema.cs | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs b/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs index bedd539b3e..f5692fab60 100644 --- a/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs +++ b/DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.IO; using Azure.DataGateway.Service.Models; using Azure.DataGateway.Service.Services; @@ -32,8 +33,10 @@ SqlGraphQLFileMetadataProvider expectedMetadataProvider $"Did not find the expected primary keys for table {tableName}"); CollectionAssert.AreEqual( - expectedTableDefinition.ForeignKeys, - actualTableDefinition.ForeignKeys, + new SortedDictionary + (expectedTableDefinition.ForeignKeys), + new SortedDictionary + (actualTableDefinition.ForeignKeys), $"Did not find the expected foreign keys for table {tableName}"); foreach ((string columnName, ColumnDefinition expectedColumnDefinition) in expectedTableDefinition.Columns) diff --git a/DataGateway.Service/Models/DatabaseSchema.cs b/DataGateway.Service/Models/DatabaseSchema.cs index 38df1fb18a..1bd16854aa 100644 --- a/DataGateway.Service/Models/DatabaseSchema.cs +++ b/DataGateway.Service/Models/DatabaseSchema.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Azure.DataGateway.Service.Authorization; namespace Azure.DataGateway.Service.Models @@ -54,6 +55,25 @@ public class ForeignKeyDefinition /// table are implicitly assumed to be the foreign key columns. /// public List ReferencingColumns { get; set; } = new(); + + public override bool Equals(object? other) + { + return Equals(other as ForeignKeyDefinition); + } + + public bool Equals(ForeignKeyDefinition? other) + { + return other != null && + ReferencedTable.Equals(other.ReferencedTable) && + Enumerable.SequenceEqual(ReferencedColumns, other.ReferencedColumns) && + Enumerable.SequenceEqual(ReferencingColumns, other.ReferencingColumns); + } + + public override int GetHashCode() + { + return HashCode.Combine( + ReferencedTable, ReferencedColumns, ReferencingColumns); + } } public class AuthorizationRule From 192263c55c4dd2643e3f3009260599a890efee7d Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 11:55:02 -0700 Subject: [PATCH 19/25] Check for additional types in EdmModelBuilder --- DataGateway.Service/MsSqlBooks.sql | 2 +- DataGateway.Service/MySqlBooks.sql | 4 ++-- DataGateway.Service/PostgreSqlBooks.sql | 2 +- DataGateway.Service/Services/EdmModelBuilder.cs | 16 ++++++++++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/DataGateway.Service/MsSqlBooks.sql b/DataGateway.Service/MsSqlBooks.sql index 5ac58ff4df..61a3527c58 100644 --- a/DataGateway.Service/MsSqlBooks.sql +++ b/DataGateway.Service/MsSqlBooks.sql @@ -74,7 +74,7 @@ CREATE TABLE stocks( CREATE TABLE stocks_price( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - instant timestamp, + instant char(10) NOT NULL, price float, PRIMARY KEY(categoryid, pieceid, instant) ); diff --git a/DataGateway.Service/MySqlBooks.sql b/DataGateway.Service/MySqlBooks.sql index 0bb6693975..8bfa1e2d00 100644 --- a/DataGateway.Service/MySqlBooks.sql +++ b/DataGateway.Service/MySqlBooks.sql @@ -65,13 +65,13 @@ CREATE TABLE stocks( categoryName varchar(100) NOT NULL, piecesAvailable bigint DEFAULT (0), piecesRequired bigint DEFAULT (0) NOT NULL, - PRIMARY KEY(categoryid,pieceid) + PRIMARY KEY(categoryid, pieceid) ); CREATE TABLE stocks_price( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - instant timestamp, + instant char(10) NOT NULL, price float, PRIMARY KEY(categoryid, pieceid, instant) ); diff --git a/DataGateway.Service/PostgreSqlBooks.sql b/DataGateway.Service/PostgreSqlBooks.sql index 8ef5a02b7f..830a561f0e 100644 --- a/DataGateway.Service/PostgreSqlBooks.sql +++ b/DataGateway.Service/PostgreSqlBooks.sql @@ -70,7 +70,7 @@ CREATE TABLE stocks( CREATE TABLE stocks_price( categoryid bigint NOT NULL, pieceid bigint NOT NULL, - instant timestamp, + instant char(10) NOT NULL, price float, PRIMARY KEY(categoryid, pieceid, instant) ); diff --git a/DataGateway.Service/Services/EdmModelBuilder.cs b/DataGateway.Service/Services/EdmModelBuilder.cs index 3cf3e57119..8db3ea5421 100644 --- a/DataGateway.Service/Services/EdmModelBuilder.cs +++ b/DataGateway.Service/Services/EdmModelBuilder.cs @@ -51,20 +51,24 @@ private EdmModelBuilder BuildEntityTypes(DatabaseSchema schema) // need to convert our column system type to an Edm type Type columnSystemType = schema.Tables[entityName].Columns[column].SystemType; EdmPrimitiveTypeKind type = EdmPrimitiveTypeKind.None; + if (columnSystemType.IsArray) + { + columnSystemType = columnSystemType.GetElementType()!; + } - switch(columnSystemType) + switch(Type.GetTypeCode(columnSystemType)) { - case string: + case TypeCode.String: type = EdmPrimitiveTypeKind.String; break; case TypeCode.Int64: type = EdmPrimitiveTypeKind.Int64; break; - case TypeCode.Byte: - type = EdmPrimitiveTypeKind.Byte; + case TypeCode.Single: + type = EdmPrimitiveTypeKind.Single; break; - case Type.GetTypeCode(byte[]): - type = EdmPrimitiveTypeKind.TimeOfDay; + case TypeCode.Double: + type = EdmPrimitiveTypeKind.Double; break; default: throw new ArgumentException($"No resolver for column type" + From cec794a01250487e27a0f88824beebd781053ef5 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 12:02:42 -0700 Subject: [PATCH 20/25] Fix formatting --- DataGateway.Service.Tests/SqlTests/SqlTestBase.cs | 6 ++++-- DataGateway.Service/Services/EdmModelBuilder.cs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs b/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs index e4f88808af..ccfd48b5cb 100644 --- a/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs +++ b/DataGateway.Service.Tests/SqlTests/SqlTestBase.cs @@ -64,7 +64,8 @@ protected static async Task InitializeTestFixture(TestContext context, string te _dbExceptionParser = new PostgresDbExceptionParser(); _queryExecutor = new QueryExecutor(config, _dbExceptionParser); _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( - config, new PostgreSqlMetadataProvider(config, _queryExecutor, _queryBuilder)); + config, + new PostgreSqlMetadataProvider(config, _queryExecutor, _queryBuilder)); break; case TestCategory.MSSQL: _queryBuilder = new MsSqlQueryBuilder(); @@ -81,7 +82,8 @@ protected static async Task InitializeTestFixture(TestContext context, string te _dbExceptionParser = new MySqlDbExceptionParser(); _queryExecutor = new QueryExecutor(config, _dbExceptionParser); _metadataStoreProvider = new SqlGraphQLFileMetadataProvider( - config, new MySqlMetadataProvider(config, _queryExecutor, _queryBuilder)); + config, + new MySqlMetadataProvider(config, _queryExecutor, _queryBuilder)); break; } diff --git a/DataGateway.Service/Services/EdmModelBuilder.cs b/DataGateway.Service/Services/EdmModelBuilder.cs index 8db3ea5421..bc8533e5d9 100644 --- a/DataGateway.Service/Services/EdmModelBuilder.cs +++ b/DataGateway.Service/Services/EdmModelBuilder.cs @@ -56,7 +56,7 @@ private EdmModelBuilder BuildEntityTypes(DatabaseSchema schema) columnSystemType = columnSystemType.GetElementType()!; } - switch(Type.GetTypeCode(columnSystemType)) + switch (Type.GetTypeCode(columnSystemType)) { case TypeCode.String: type = EdmPrimitiveTypeKind.String; From 27e94af814b4ae3a4b99d37e9c1137b30b75aae8 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 12:07:37 -0700 Subject: [PATCH 21/25] sql-config PreserveNewest --- .../Azure.DataGateway.Service.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj b/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj index 4bb00a563e..cb940cf911 100644 --- a/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj +++ b/DataGateway.Service.Tests/Azure.DataGateway.Service.Tests.csproj @@ -28,7 +28,7 @@ - Always + PreserveNewest From bc7192cc61527f0c81d7458e10ee6cbc2f02f098 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Wed, 13 Apr 2022 12:15:22 -0700 Subject: [PATCH 22/25] Remove unnecessary HttpVerbs --- DataGateway.Service.Tests/sql-config-test.json | 17 ----------------- .../MetadataProviders/SqlMetadataProvider.cs | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/DataGateway.Service.Tests/sql-config-test.json b/DataGateway.Service.Tests/sql-config-test.json index 8f7f87aaca..dd66bea984 100644 --- a/DataGateway.Service.Tests/sql-config-test.json +++ b/DataGateway.Service.Tests/sql-config-test.json @@ -183,23 +183,6 @@ "ReferencedColumns": [ "id" ], "ReferencingColumns": [ "book_id" ] } - }, - "HttpVerbs": { - "GET": { - "AuthorizationType": "Anonymous" - }, - "POST": { - "AuthorizationType": "Authenticated" - }, - "DELETE": { - "Authorization": "Authenticated" - }, - "PUT": { - "Authorization": "Authenticated" - }, - "PATCH": { - "Authorization": "Authenticated" - } } }, "authors": { diff --git a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs index 41d184abaa..00fe1b721c 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs @@ -103,7 +103,7 @@ public async Task PopulateForeignKeyDefinitionAsync( // Build the array storing all the schemaNames, for now the defaultSchemaName. string[] schemaNames = Enumerable.Range(1, tables.Count).Select(x => defaultSchemaName).ToArray(); - // Build the parameters for dictionary for the foreign key info query + // Build the parameters dictionary for the foreign key info query // consisting of all schema names and table names. Dictionary parameters = GetForeignKeyQueryParams(schemaNames, tables.Keys.ToArray()); From c6751dfa69f53c933d1b0dacedab2dc0a12b9dd1 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 19 Apr 2022 19:34:48 -0700 Subject: [PATCH 23/25] Address review comments --- .../Resolvers/BaseSqlQueryBuilder.cs | 57 +++++++++++-------- .../Resolvers/MySqlQueryBuilder.cs | 34 ++++++----- .../MySqlMetadataProvider.cs | 4 +- .../MetadataProviders/SqlMetadataProvider.cs | 9 +-- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs index b08ff5a6b1..7637fe4c03 100644 --- a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs @@ -273,33 +273,42 @@ public virtual string BuildForeignKeyInfoQuery(int numberOfParameters) CreateParams(kindOfParam: TABLE_NAME_PARAM, numberOfParameters); string tableSchemaParamsForInClause = string.Join(", @", schemaNameParams); string tableNameParamsForInClause = string.Join(", @", tableNameParams); - return $"" + - $"SELECT " + - $"ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, " + - $"ReferencingColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, " + - $"ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, " + - $"ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, " + - $"ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} " + - $"FROM " + - $"INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints " + - $"INNER JOIN " + - $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage " + - $"ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG " + - $"AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA " + - $"AND ReferentialConstraints.CONSTRAINT_NAME = ReferencingColumnUsage.CONSTRAINT_NAME " + - $"INNER JOIN " + - $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencedColumnUsage " + - $"ON ReferentialConstraints.UNIQUE_CONSTRAINT_CATALOG = ReferencedColumnUsage.CONSTRAINT_CATALOG " + - $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA " + - $"AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME " + - $"WHERE " + - $"ReferencingColumnUsage.TABLE_SCHEMA IN (@{tableSchemaParamsForInClause})" + - $"AND ReferencingColumnUsage.TABLE_NAME IN (@{tableNameParamsForInClause})" + - $"AND ReferencingColumnUsage.ORDINAL_POSITION = ReferencedColumnUsage.ORDINAL_POSITION;"; + + // The view REFERENTIAL_CONSTRAINTS has a row for each referential key CONSTRAINT_NAME and + // its corresponding UNIQUE_CONSTRAINT_NAME to which it references. + // These are only constraint names so we need to join with the view KEY_COLUMN_USAGE to get the + // constraint columns - one inner join for the columns from the 'Referencing table' + // and the other join for the columns from the 'Referenced Table'. + string foreignKeyQuery = $@" + SELECT + ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, + ReferencingColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, + ReferencingColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, + ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, + ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} + FROM + INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints + INNER JOIN + INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage + ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG + AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA + AND ReferentialConstraints.CONSTRAINT_NAME = ReferencingColumnUsage.CONSTRAINT_NAME + INNER JOIN + INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencedColumnUsage + ON ReferentialConstraints.UNIQUE_CONSTRAINT_CATALOG = ReferencedColumnUsage.CONSTRAINT_CATALOG + AND ReferentialConstraints.UNIQUE_CONSTRAINT_SCHEMA = ReferencedColumnUsage.CONSTRAINT_SCHEMA + AND ReferentialConstraints.UNIQUE_CONSTRAINT_NAME = ReferencedColumnUsage.CONSTRAINT_NAME + AND ReferencingColumnUsage.ORDINAL_POSITION = ReferencedColumnUsage.ORDINAL_POSITION + WHERE + ReferencingColumnUsage.TABLE_SCHEMA IN (@{tableSchemaParamsForInClause}) + AND ReferencingColumnUsage.TABLE_NAME IN (@{tableNameParamsForInClause})"; + + Console.WriteLine($"Foreign Key Query: {foreignKeyQuery}"); + return foreignKeyQuery; } /// - public string[] CreateParams(string kindOfParam, int numberOfParameters) + public static string[] CreateParams(string kindOfParam, int numberOfParameters) { return Enumerable.Range(0, numberOfParameters).Select(i => kindOfParam + i).ToArray(); } diff --git a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs index e337b5636e..387a5eceff 100644 --- a/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/MySqlQueryBuilder.cs @@ -128,20 +128,26 @@ public override string BuildForeignKeyInfoQuery(int numberOfParameters) string[] tableNameParams = CreateParams(TABLE_NAME_PARAM, numberOfParameters); string tableSchemaParamsForInClause = string.Join(", @", databaseNameParams); string tableNameParamsForInClause = string.Join(", @", tableNameParams); - return $"" + - $"SELECT " + - $"CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, " + - $"TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, " + - $"COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, " + - $"REFERENCED_TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, " + - $"REFERENCED_COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} " + - $"FROM " + - $"INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + - $"WHERE " + - $"TABLE_SCHEMA IN (@{tableSchemaParamsForInClause}) " + - $"AND TABLE_NAME IN (@{tableNameParamsForInClause}) " + - $"AND REFERENCED_TABLE_NAME IS NOT NULL " + - $"AND REFERENCED_COLUMN_NAME IS NOT NULL;"; + + // For MySQL, the view KEY_COLUMN_USAGE provides all the information we need + // so there is no need to join with any other view. + string foreignKeyQuery = $@" + SELECT + CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))}, + TABLE_NAME {QuoteIdentifier(nameof(TableDefinition))}, + COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencingColumns))}, + REFERENCED_TABLE_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedTable))}, + REFERENCED_COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))} + FROM + INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE + TABLE_SCHEMA IN (@{tableSchemaParamsForInClause}) + AND TABLE_NAME IN (@{tableNameParamsForInClause}) + AND REFERENCED_TABLE_NAME IS NOT NULL + AND REFERENCED_COLUMN_NAME IS NOT NULL;"; + + Console.WriteLine($"Foreign Key Query is : {foreignKeyQuery}"); + return foreignKeyQuery; } /// diff --git a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs index 162c281613..58f36a2c41 100644 --- a/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/MySqlMetadataProvider.cs @@ -70,11 +70,11 @@ protected override async Task GetColumnsAsync( Dictionary parameters = new(); string[] databaseNameParams = - SqlQueryBuilder!.CreateParams( + BaseSqlQueryBuilder.CreateParams( kindOfParam: MySqlQueryBuilder.DATABASE_NAME_PARAM, schemaNames.Count()); string[] tableNameParams = - SqlQueryBuilder!.CreateParams( + BaseSqlQueryBuilder.CreateParams( kindOfParam: BaseSqlQueryBuilder.TABLE_NAME_PARAM, tableNames.Count()); diff --git a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs index 00fe1b721c..5bcb73597d 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs @@ -32,7 +32,7 @@ public class SqlMetadataProvider : ISqlMeta protected string ConnectionString { get; init; } // nullable since Mock tests don't need this. - protected IQueryBuilder? SqlQueryBuilder { get; init; } + protected IQueryBuilder SqlQueryBuilder { get; init; } protected DataSet EntitiesDataSet { get; init; } @@ -54,6 +54,7 @@ public SqlMetadataProvider() { ConnectionString = new(string.Empty); EntitiesDataSet = new(); + SqlQueryBuilder = new MsSqlQueryBuilder(); } /// @@ -98,7 +99,7 @@ public async Task PopulateForeignKeyDefinitionAsync( { // Build the query required to get the foreign key information. string queryForForeignKeyInfo = - ((BaseSqlQueryBuilder)SqlQueryBuilder!).BuildForeignKeyInfoQuery(tables.Count); + ((BaseSqlQueryBuilder)SqlQueryBuilder).BuildForeignKeyInfoQuery(tables.Count); // Build the array storing all the schemaNames, for now the defaultSchemaName. string[] schemaNames = Enumerable.Range(1, tables.Count).Select(x => defaultSchemaName).ToArray(); @@ -268,11 +269,11 @@ protected void PopulateColumnDefinitionWithHasDefault( { Dictionary parameters = new(); string[] schemaNameParams = - SqlQueryBuilder!.CreateParams( + BaseSqlQueryBuilder.CreateParams( kindOfParam: BaseSqlQueryBuilder.SCHEMA_NAME_PARAM, schemaNames.Count()); string[] tableNameParams = - SqlQueryBuilder!.CreateParams( + BaseSqlQueryBuilder.CreateParams( kindOfParam: BaseSqlQueryBuilder.TABLE_NAME_PARAM, tableNames.Count()); From 16a3c37ed71f8f30208345681f5e69d013d0c43b Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 19 Apr 2022 20:12:20 -0700 Subject: [PATCH 24/25] Remove CreateParams from the interface --- DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs | 10 +++++++++- DataGateway.Service/Resolvers/IQueryBuilder.cs | 11 ----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs index 1ec8f897e4..af17195218 100644 --- a/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/BaseSqlQueryBuilder.cs @@ -311,7 +311,15 @@ ReferencingColumnUsage.TABLE_SCHEMA IN (@{tableSchemaParamsForInClause}) return foreignKeyQuery; } - /// + /// + /// Creates a list of named parameters with incremental suffixes + /// starting from 0 to numberOfParameters - 1. + /// e.g. tableName0, tableName1 + /// + /// The kind of parameter being created acting + /// as the prefix common to all parameters. + /// The number of parameters to create. + /// The created list public static string[] CreateParams(string kindOfParam, int numberOfParameters) { return Enumerable.Range(0, numberOfParameters).Select(i => kindOfParam + i).ToArray(); diff --git a/DataGateway.Service/Resolvers/IQueryBuilder.cs b/DataGateway.Service/Resolvers/IQueryBuilder.cs index 47a74f2da2..8a1c8039af 100644 --- a/DataGateway.Service/Resolvers/IQueryBuilder.cs +++ b/DataGateway.Service/Resolvers/IQueryBuilder.cs @@ -41,16 +41,5 @@ public interface IQueryBuilder /// number of parameters. /// public string BuildForeignKeyInfoQuery(int numberOfParameters); - - /// - /// Creates a list of named parameters with incremental suffixes - /// starting from 0 to numberOfParameters - 1. - /// e.g. tableName0, tableName1 - /// - /// The kind of parameter being created acting - /// as the prefix common to all parameters. - /// The number of parameters to create. - /// The created list - public string[] CreateParams(string kindOfParam, int numberOfParameters); } } From 941bb47dececd479cd853f554fa27520e8851bb6 Mon Sep 17 00:00:00 2001 From: Aniruddh Munde Date: Tue, 19 Apr 2022 20:44:34 -0700 Subject: [PATCH 25/25] Added a TODO comment to track removal of nullability of _queryExecutor property --- .../Services/MetadataProviders/SqlMetadataProvider.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs index 5bcb73597d..9f276e4f45 100644 --- a/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs +++ b/DataGateway.Service/Services/MetadataProviders/SqlMetadataProvider.cs @@ -23,6 +23,8 @@ public class SqlMetadataProvider : ISqlMeta where CommandT : DbCommand, new() { // nullable since Mock tests do not need it. + // TODO: Refactor the Mock tests to remove the nullability here + // once the runtime config is implemented tracked by #353. private readonly IQueryExecutor? _queryExecutor; private const int NUMBER_OF_RESTRICTIONS = 4;