From 7a1b26293ebc7d51aef20cfc282a930119cc2d2b Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 10 Jun 2022 14:29:15 -0700 Subject: [PATCH 01/16] Add parsing for env variables to config + tests, newtonsoft needed for parsing --- .../Azure.DataGateway.Config.csproj | 1 + DataGateway.Config/RuntimeConfigPath.cs | 103 ++++- .../Unittests/RuntimeConfigPathUnitTests.cs | 437 ++++++++++++++++++ 3 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs diff --git a/DataGateway.Config/Azure.DataGateway.Config.csproj b/DataGateway.Config/Azure.DataGateway.Config.csproj index e55bacd831..d046e3e7c8 100644 --- a/DataGateway.Config/Azure.DataGateway.Config.csproj +++ b/DataGateway.Config/Azure.DataGateway.Config.csproj @@ -8,6 +8,7 @@ + diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 108ad33581..9f9d058af3 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -1,3 +1,8 @@ +using System.Text; +using System.Text.RegularExpressions; +using Azure.DataGateway.Service.Exceptions; +using Newtonsoft.Json; + namespace Azure.DataGateway.Config { /// @@ -31,7 +36,7 @@ public void SetRuntimeConfigValue() { if (File.Exists(ConfigFileName)) { - runtimeConfigJson = File.ReadAllText(ConfigFileName); + runtimeConfigJson = ParseConfigJsonAndReplaceEnvVariables(File.ReadAllText(ConfigFileName)); } else { @@ -51,6 +56,102 @@ public void SetRuntimeConfigValue() } } + /// + /// Parse Json and replace @env('ENVIRONMENT_VARIABLE_NAME') with + /// the environment variable that corresponds to ENVIRONMENT_VARIABLE_NAME. + /// If no environment variable is found with that name, throw exception. + /// + /// Json string representing the runtime config file. + /// Parsed json string. + public static string? ParseConfigJsonAndReplaceEnvVariables(string json) + { + StringBuilder stringBuilder = new(); + StringWriter stringWriter = new(stringBuilder); + JsonTextReader reader = new(new StringReader(json)); + JsonTextWriter writer = new(stringWriter) + { + Formatting = Formatting.Indented + }; + + // @env : match @env(' + // .*? : lazy match any character except newline 0 or more times + // (?='\)) : look ahead for ') which will combine with our lazy match + // ie: in @env('hello')goodbye') we match @env('hello') + // '\) : consume the ') into the match (look ahead doesn't capture) + // This pattern lazy matches any string that starts with @env(' and ends with ') + // ie: fooBAR@env('hello-world')bash)FOO') match: @env('hello-world') @env(('hello-world') + string envPattern = @"@env\('.*?(?='\))'\)"; + + // The approach for parsing is to re-write the Json to a new string + // as we read, using regex.replace for the matches we get from our + // pattern. We call a helper function for each match that handles + // getting the environment variable for replacement. + while (reader.Read()) + { + if (reader.Value != null) + { + switch (reader.TokenType) + { + case JsonToken.PropertyName: + writer.WritePropertyName(reader.Value.ToString()!); + break; + case JsonToken.String: + string valueToWrite = Regex.Replace(reader.Value.ToString()!, envPattern, new MatchEvaluator(ReplaceWithEnvVariable)); + writer.WriteValue(valueToWrite); + break; + default: + writer.WriteValue(reader.Value); + break; + } + } + else + { + switch (reader.TokenType) + { + case JsonToken.StartObject: + writer.WriteStartObject(); + break; + case JsonToken.StartArray: + writer.WriteStartArray(); + break; + case JsonToken.EndArray: + writer.WriteEndArray(); + break; + case JsonToken.EndObject: + writer.WriteEndObject(); + break; + } + } + } + + return stringBuilder.ToString(); + } + + /// + /// Retrieves the name of the environment variable + /// and then returns the environment variable associated + /// with that name, throwing an exception if none is found. + /// + /// The match holding the environment variable name. + /// The environment variable associate with the provided name. + /// + private static string ReplaceWithEnvVariable(Match match) + { + // [^@env\(] : any substring that is not @env( + // .* : any char except newline any number of times + // (?=\)) : look ahead for end char of ) + // This pattern greedy matches all characters that are not a part of @env() + // ie: @env('hello@env('goodbye')world') match: 'hello@env('goodbye')world' + string innerPattern = @"[^@env\(].*(?=\))"; + + // strip's first and last characters, ie: '''hello'' --> ''hello' + string envName = Regex.Match(match.Value, innerPattern).Value[1..^1]; + string? envValue = Environment.GetEnvironmentVariable(envName); + return envValue is not null ? envValue : throw new DataGatewayException(message: $"Environmental Variable, {envName}, not found.", + statusCode: System.Net.HttpStatusCode.ServiceUnavailable, + subStatusCode: DataGatewayException.SubStatusCodes.ErrorInInitialization); + } + /// /// Returns whether or not this RuntimeConfigPath /// is in developer mode. diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs new file mode 100644 index 0000000000..bf134d36b6 --- /dev/null +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -0,0 +1,437 @@ +using System; +using System.Data; +using Azure.DataGateway.Config; +using Azure.DataGateway.Service.Exceptions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; + +namespace Azure.DataGateway.Service.Tests.UnitTests +{ + /// + /// Unit tests for the environment variable + /// parser for the runtime configuration. These + /// tests verify that we parse the config correcltly + /// when replacing envinroment variables. Also verify + /// we throw the right exception when environment + /// variable names not found. + /// to retreive schema. + /// + [TestClass, TestCategory(TestCategory.MSSQL)] + public class RuntimeConfigPathUnitTests + { + #region Positive Tests + + /// + /// Test valid cases for parsing the runtime config. + /// These cases have string close to the pattern we + /// match when looking to replace parts of the config, + /// strings that match said pattern, and other edge + /// cases to reveal if the pattern matching is working. + /// The pattern we look to match is @env('') where we take + /// what is inside of the ''. The match is then + /// used to get the associated environment variable. + /// + /// Replacement used as key to get environment variable. + /// Replacement value. + [DataTestMethod] + [DataRow(new string[] { "@env(')", "@env()", "@env(')'@env('()", "@env('@env()'", "@@eennvv((''''))" }, + new object[] + { + new string[] { "@env(')", "@env()", "@env(')'@env('()", "@env('@env()'", "@@eennvv((''''))" } + }, + DisplayName = "Replacement strings that won't match.")] + [DataRow(new string[] { "@env('envVarName')", "@env(@env('envVarName'))", "@en@env('envVarName')", "@env'()@env'@env('envVarName')')')" }, + new object[] + { + new string[] { "envVarValue", "@env(envVarValue)", "@enenvVarValue", "@env'()@env'envVarValue')')" } + }, + DisplayName = "Replacement strings that match.")] + [DataRow(new string[] { "@env(')", "@env()", "@env('envVarName')", "@env(''envVarName')", "@env('envVarName'')", "@env(''envVarName'')" }, + new object[] + { + new string[] { "@env(')", "@env()", "envVarValue", "_envVarValue", "envVarValue_", "_envVarValue_" } + }, + DisplayName = "Replacement strings with some matches.")] + public void CheckConfigEnvParsingTest(string[] repKeys, string[] repValues) + { + SetEnvVariables(); + string expectedJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(GetModifiedJsonString(repValues)); + string actualJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(GetModifiedJsonString(repKeys)); + JObject expected = JObject.Parse(expectedJson); + JObject actual = JObject.Parse(actualJson); + Assert.IsTrue(JToken.DeepEquals(expected, actual)); + } + + #endregion Positive Tests + + #region Negative Tests + + /// + /// When we have a match that does not correspond + /// to a valid environment variable we throw an exception. + /// This tests verifies this happens correctly. + /// + /// A match that is not a valid environment variable name. + [DataTestMethod] + [DataRow("")] + [DataRow("fooBARbaz")] + [DataRow("''envVarName")] + [DataRow("''envVarName'")] + [DataRow("envVarName''")] + [DataRow("''envVarName''")] + public void CheckConfigEnvParsingThrowExceptions(string invalidEnvVarName) + { + string json =@"{ ""foo"" : ""@env('envVarName'), @env('" + invalidEnvVarName + @"')"" }"; + SetEnvVariables(); + Assert.ThrowsException(() => RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(json)); + } + + #endregion Negative Tests + + #region Helper Functions + + /// + /// Setup some environment variables. + /// + private static void SetEnvVariables() + { + Environment.SetEnvironmentVariable($"envVarName", $"envVarValue"); + Environment.SetEnvironmentVariable($"'envVarName", $"_envVarValue"); + Environment.SetEnvironmentVariable($"envVarName'", $"envVarValue_"); + Environment.SetEnvironmentVariable($"'envVarName'", $"_envVarValue_"); + } + + /// + /// Modify the json string with the replacements provided. + /// This function cycles through the string array in a circular + /// fasion. + /// + /// Replacement strings. + /// Json string with replacements. + public static string GetModifiedJsonString(string[] reps) + { + int index = 0; + return +@"{ + ""$schema"": "".. /../project-hawaii/playground/hawaii.draft-01.schema.json"", + ""data-source"": { + ""database-type"": """ + reps[index % reps.Length] + @""", + ""connection-string"": ""server=datagateway;database=" + reps[++index % reps.Length] + @";uid=" + reps[++index % reps.Length] + @";Password=" + reps[++index % reps.Length] + @";Allow User Variables=true"", + ""resolver-config-file"": """ + reps[++index % reps.Length] + @""" + }, + ""runtime"": { + ""rest"": { + ""enabled"": """ + reps[++index % reps.Length] + @""", + ""path"": ""/" + reps[++index % reps.Length] + @""" + }, + ""graphql"": { + ""enabled"": true, + ""path"": """ + reps[++index % reps.Length] + @""", + ""allow-introspection"": true + }, + ""host"": { + ""mode"": """ + reps[++index % reps.Length] + @""", + ""cors"": { + ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""" ], + ""allow-credentials"": """ + reps[++index % reps.Length] + @""" + }, + ""authentication"": { + ""provider"": """ + reps[++index % reps.Length] + @""", + ""jwt"": { + ""audience"": """", + ""issuer"": """", + ""issuer-key"": """ + reps[++index % reps.Length] + @""" + } + } + } +}, + ""entities"": { + ""Publisher"": { + ""source"": """ + reps[++index % reps.Length] + @"." + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Stock"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", ""update"" ] + } + ], + ""relationships"": { + ""comics"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""", + ""source.fields"": [ ""categoryName"" ], + ""target.fields"": [ """ + reps[++index % reps.Length] + @""" ] + } + } + }, + ""Book"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""update"", """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + ""publishers"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""websiteplacement"": { + ""cardinality"": ""one"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""reviews"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""authors"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": ""Author"", + ""linking.object"": ""book_author_link"", + ""linking.source.fields"": [ ""book_id"" ], + ""linking.target.fields"": [ """ + reps[++index % reps.Length] + @""" ] + } + }, + ""mappings"": { + ""id"": """ + reps[++index % reps.Length] + @""", + ""title"": """ + reps[++index % reps.Length] + @""" + } + }, + ""BookWebsitePlacement"": { + ""source"": ""book_website_placements"", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ + """ + reps[++index % reps.Length] + @""", + """ + reps[++index % reps.Length] + @""", + { + ""action"": ""delete"", + ""policy"": { + ""database"": ""@claims.id eq @item.id"" + }, + ""fields"": { + ""include"": [ ""*"" ], + ""exclude"": [ """ + reps[++index % reps.Length] + @""" ] + } + } + ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""one"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Author"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""", + ""linking.object"": ""book_author_link"" + } + } + }, + ""Review"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Comic"": { + ""source"": ""comics"", + ""rest"": true, + ""graphql"": false, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", """ + reps[++index % reps.Length] + @""" ] + } + ] + }, + ""Broker"": { + ""source"": ""brokers"", + ""graphql"": false, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ] + }, + ""WebsiteUser"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": false, + ""permissions"": [] + }, + ""SupportedType"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": false, + ""permissions"": [] + }, + ""stocks_price"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [] + }, + ""Tree"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] + } + ], + ""mappings"": { + ""species"": ""Scientific Name"", + ""region"": ""United State's " + reps[++index % reps.Length] + @""" + } + }, + ""Shrub"": { + ""source"": ""trees"", + ""rest"": true, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""create"", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] + } + ], + ""mappings"": { + ""species"": """ + reps[++index % reps.Length] + @""" + } + }, + ""Fungus"": { + ""source"": ""fungi"", + ""rest"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] + } + ], + ""mappings"": { + ""spores"": ""hazards"" + } + }, + ""books_view_all"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + } + }, + ""stocks_view_selected"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ ""read"" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + } + }, + ""books_publishers_view_composite"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + } + } + } +} +"; + } + + #endregion Helper Functions + } +} From 201d9f423b4008e99390d17d03c790f9d6be5522 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 10 Jun 2022 14:42:10 -0700 Subject: [PATCH 02/16] typos --- DataGateway.Config/RuntimeConfigPath.cs | 2 +- .../Unittests/RuntimeConfigPathUnitTests.cs | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 9f9d058af3..89d3d0316f 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -133,7 +133,7 @@ public void SetRuntimeConfigValue() /// with that name, throwing an exception if none is found. /// /// The match holding the environment variable name. - /// The environment variable associate with the provided name. + /// The environment variable associated with the provided name. /// private static string ReplaceWithEnvVariable(Match match) { diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index bf134d36b6..496ed232dd 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -10,11 +10,10 @@ namespace Azure.DataGateway.Service.Tests.UnitTests /// /// Unit tests for the environment variable /// parser for the runtime configuration. These - /// tests verify that we parse the config correcltly - /// when replacing envinroment variables. Also verify + /// tests verify that we parse the config correctly + /// when replacing environment variables. Also verify /// we throw the right exception when environment - /// variable names not found. - /// to retreive schema. + /// variable names are not found. /// [TestClass, TestCategory(TestCategory.MSSQL)] public class RuntimeConfigPathUnitTests @@ -23,12 +22,12 @@ public class RuntimeConfigPathUnitTests /// /// Test valid cases for parsing the runtime config. - /// These cases have string close to the pattern we + /// These cases have strings close to the pattern we /// match when looking to replace parts of the config, /// strings that match said pattern, and other edge /// cases to reveal if the pattern matching is working. /// The pattern we look to match is @env('') where we take - /// what is inside of the ''. The match is then + /// what is inside of the '', ie: @env(''). The match is then /// used to get the associated environment variable. /// /// Replacement used as key to get environment variable. @@ -69,7 +68,7 @@ public void CheckConfigEnvParsingTest(string[] repKeys, string[] repValues) /// /// When we have a match that does not correspond /// to a valid environment variable we throw an exception. - /// This tests verifies this happens correctly. + /// These tests verify this happens correctly. /// /// A match that is not a valid environment variable name. [DataTestMethod] From a1cc84e54b52582c823d2991cef3ae1c957eb07a Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 10 Jun 2022 14:46:34 -0700 Subject: [PATCH 03/16] missing spac after = --- .../Unittests/RuntimeConfigPathUnitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index 496ed232dd..f775a84130 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -80,7 +80,7 @@ public void CheckConfigEnvParsingTest(string[] repKeys, string[] repValues) [DataRow("''envVarName''")] public void CheckConfigEnvParsingThrowExceptions(string invalidEnvVarName) { - string json =@"{ ""foo"" : ""@env('envVarName'), @env('" + invalidEnvVarName + @"')"" }"; + string json = @"{ ""foo"" : ""@env('envVarName'), @env('" + invalidEnvVarName + @"')"" }"; SetEnvVariables(); Assert.ThrowsException(() => RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(json)); } From eadd129f692d75612971fc6eb7db457326e7ee73 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 10 Jun 2022 14:57:19 -0700 Subject: [PATCH 04/16] typo in a comment --- DataGateway.Config/RuntimeConfigPath.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 89d3d0316f..e302ad9b46 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -79,7 +79,7 @@ public void SetRuntimeConfigValue() // ie: in @env('hello')goodbye') we match @env('hello') // '\) : consume the ') into the match (look ahead doesn't capture) // This pattern lazy matches any string that starts with @env(' and ends with ') - // ie: fooBAR@env('hello-world')bash)FOO') match: @env('hello-world') @env(('hello-world') + // ie: fooBAR@env('hello-world')bash)FOO') match: @env('hello-world') string envPattern = @"@env\('.*?(?='\))'\)"; // The approach for parsing is to re-write the Json to a new string From b25d38a37130e4bd0e6954b81b5e3eb0332c9e37 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Fri, 10 Jun 2022 15:46:18 -0700 Subject: [PATCH 05/16] is not null prefered --- DataGateway.Config/RuntimeConfigPath.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index e302ad9b46..4ead0069ea 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -88,7 +88,7 @@ public void SetRuntimeConfigValue() // getting the environment variable for replacement. while (reader.Read()) { - if (reader.Value != null) + if (reader.Value is not null) { switch (reader.TokenType) { From 1df97ea19e1563dd8d97e56fa23007f761b33ece Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 14 Jun 2022 21:56:02 -0700 Subject: [PATCH 06/16] null case added, better formatting for test string, using keyword for jsonwrite/read --- DataGateway.Config/RuntimeConfigPath.cs | 14 +- .../Unittests/RuntimeConfigPathUnitTests.cs | 158 +++++++++--------- 2 files changed, 88 insertions(+), 84 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 4ead0069ea..196530377d 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -67,8 +67,8 @@ public void SetRuntimeConfigValue() { StringBuilder stringBuilder = new(); StringWriter stringWriter = new(stringBuilder); - JsonTextReader reader = new(new StringReader(json)); - JsonTextWriter writer = new(stringWriter) + using JsonTextReader reader = new(new StringReader(json)); + using JsonTextWriter writer = new(stringWriter) { Formatting = Formatting.Indented }; @@ -120,6 +120,9 @@ public void SetRuntimeConfigValue() case JsonToken.EndObject: writer.WriteEndObject(); break; + case JsonToken.Null: + writer.WriteNull(); + break; } } } @@ -147,9 +150,10 @@ private static string ReplaceWithEnvVariable(Match match) // strip's first and last characters, ie: '''hello'' --> ''hello' string envName = Regex.Match(match.Value, innerPattern).Value[1..^1]; string? envValue = Environment.GetEnvironmentVariable(envName); - return envValue is not null ? envValue : throw new DataGatewayException(message: $"Environmental Variable, {envName}, not found.", - statusCode: System.Net.HttpStatusCode.ServiceUnavailable, - subStatusCode: DataGatewayException.SubStatusCodes.ErrorInInitialization); + return envValue is not null ? envValue : + throw new DataGatewayException(message: $"Environmental Variable, {envName}, not found.", + statusCode: System.Net.HttpStatusCode.ServiceUnavailable, + subStatusCode: DataGatewayException.SubStatusCodes.ErrorInInitialization); } /// diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index f775a84130..2304361e58 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -120,134 +120,134 @@ public static string GetModifiedJsonString(string[] reps) }, ""runtime"": { ""rest"": { - ""enabled"": """ + reps[++index % reps.Length] + @""", + ""enabled"": """ + reps[++index % reps.Length] + @""", ""path"": ""/" + reps[++index % reps.Length] + @""" }, ""graphql"": { - ""enabled"": true, + ""enabled"": true, ""path"": """ + reps[++index % reps.Length] + @""", ""allow-introspection"": true }, ""host"": { - ""mode"": """ + reps[++index % reps.Length] + @""", + ""mode"": """ + reps[++index % reps.Length] + @""", ""cors"": { - ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""" ], + ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""" ], ""allow-credentials"": """ + reps[++index % reps.Length] + @""" }, ""authentication"": { - ""provider"": """ + reps[++index % reps.Length] + @""", + ""provider"": """ + reps[++index % reps.Length] + @""", ""jwt"": { ""audience"": """", ""issuer"": """", ""issuer-key"": """ + reps[++index % reps.Length] + @""" } - } + } } -}, + }, ""entities"": { ""Publisher"": { - ""source"": """ + reps[++index % reps.Length] + @"." + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @"." + reps[++index % reps.Length] + @""", ""rest"": """ + reps[++index % reps.Length] + @""", ""graphql"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": ""authenticated"", + ""role"": ""authenticated"", ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] } ], ""relationships"": { - ""books"": { - ""cardinality"": ""many"", + ""books"": { + ""cardinality"": ""many"", ""target.entity"": """ + reps[++index % reps.Length] + @""" - } } + } }, ""Stock"": { - ""source"": """ + reps[++index % reps.Length] + @""", - ""rest"": true, + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": null, ""graphql"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": ""authenticated"", + ""role"": ""authenticated"", ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", ""update"" ] } ], ""relationships"": { - ""comics"": { - ""cardinality"": ""many"", + ""comics"": { + ""cardinality"": ""many"", ""target.entity"": """ + reps[++index % reps.Length] + @""", ""source.fields"": [ ""categoryName"" ], ""target.fields"": [ """ + reps[++index % reps.Length] + @""" ] } - } + } }, ""Book"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": ""authenticated"", + ""role"": ""authenticated"", ""actions"": [ """ + reps[++index % reps.Length] + @""", ""update"", """ + reps[++index % reps.Length] + @""" ] } ], ""relationships"": { - ""publishers"": { - ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""publishers"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", ""target.entity"": """ + reps[++index % reps.Length] + @""" - }, + }, ""websiteplacement"": { - ""cardinality"": ""one"", + ""cardinality"": ""one"", ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""reviews"": { - ""cardinality"": ""many"", + ""cardinality"": ""many"", ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""authors"": { - ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""cardinality"": """ + reps[++index % reps.Length] + @""", ""target.entity"": ""Author"", ""linking.object"": ""book_author_link"", ""linking.source.fields"": [ ""book_id"" ], ""linking.target.fields"": [ """ + reps[++index % reps.Length] + @""" ] } - }, + }, ""mappings"": { - ""id"": """ + reps[++index % reps.Length] + @""", + ""id"": """ + reps[++index % reps.Length] + @""", ""title"": """ + reps[++index % reps.Length] + @""" } }, ""BookWebsitePlacement"": { - ""source"": ""book_website_placements"", + ""source"": ""book_website_placements"", ""rest"": """ + reps[++index % reps.Length] + @""", ""graphql"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""", { - ""action"": ""delete"", + ""action"": ""delete"", ""policy"": { - ""database"": ""@claims.id eq @item.id"" + ""database"": ""@claims.id eq @item.id"" }, ""fields"": { - ""include"": [ ""*"" ], + ""include"": [ ""*"" ], ""exclude"": [ """ + reps[++index % reps.Length] + @""" ] } } @@ -255,54 +255,54 @@ public static string GetModifiedJsonString(string[] reps) } ], ""relationships"": { - ""books"": { - ""cardinality"": ""one"", + ""books"": { + ""cardinality"": ""one"", ""target.entity"": """ + reps[++index % reps.Length] + @""" - } } + } }, ""Author"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": true, ""graphql"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ ""read"" ] } ], ""relationships"": { - ""books"": { - ""cardinality"": ""many"", + ""books"": { + ""cardinality"": ""many"", ""target.entity"": """ + reps[++index % reps.Length] + @""", ""linking.object"": ""book_author_link"" - } } + } }, ""Review"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": true, ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] } ], ""relationships"": { - ""books"": { - ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""books"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", ""target.entity"": """ + reps[++index % reps.Length] + @""" - } } + } }, ""Comic"": { - ""source"": ""comics"", + ""source"": ""comics"", ""rest"": true, - ""graphql"": false, + ""graphql"": null, ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", - ""actions"": [ ""read"" ] + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ null ] }, { ""role"": ""authenticated"", @@ -311,81 +311,81 @@ public static string GetModifiedJsonString(string[] reps) ] }, ""Broker"": { - ""source"": ""brokers"", + ""source"": ""brokers"", ""graphql"": false, ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] } ] }, ""WebsiteUser"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": false, ""permissions"": [] }, ""SupportedType"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": false, ""permissions"": [] }, ""stocks_price"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": """ + reps[++index % reps.Length] + @""", ""permissions"": [] }, ""Tree"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": """ + reps[++index % reps.Length] + @""", ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] } ], ""mappings"": { - ""species"": ""Scientific Name"", + ""species"": ""Scientific Name"", ""region"": ""United State's " + reps[++index % reps.Length] + @""" } }, ""Shrub"": { - ""source"": ""trees"", + ""source"": ""trees"", ""rest"": true, ""permissions"": [ { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ ""create"", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] } ], ""mappings"": { - ""species"": """ + reps[++index % reps.Length] + @""" + ""species"": """ + reps[++index % reps.Length] + @""" } }, ""Fungus"": { - ""source"": ""fungi"", + ""source"": ""fungi"", ""rest"": true, ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] } ], ""mappings"": { - ""spores"": ""hazards"" + ""spores"": ""hazards"" } }, ""books_view_all"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": true, ""graphql"": true, ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ ""read"" ] } ], @@ -393,38 +393,38 @@ public static string GetModifiedJsonString(string[] reps) } }, ""stocks_view_selected"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": true, ""graphql"": true, ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ ""read"" ] }, { - ""role"": """ + reps[++index % reps.Length] + @""", + ""role"": """ + reps[++index % reps.Length] + @""", ""actions"": [ ""read"" ] } ], ""relationships"": { - } + } }, ""books_publishers_view_composite"": { - ""source"": """ + reps[++index % reps.Length] + @""", + ""source"": """ + reps[++index % reps.Length] + @""", ""rest"": true, ""graphql"": true, ""permissions"": [ { - ""role"": ""anonymous"", + ""role"": ""anonymous"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] }, { - ""role"": ""authenticated"", + ""role"": ""authenticated"", ""actions"": [ """ + reps[++index % reps.Length] + @""" ] } ], ""relationships"": { - } + } } } } From 81b5762fbe22bb07bdba61e875424e4a963acbe2 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Wed, 15 Jun 2022 16:06:14 -0700 Subject: [PATCH 07/16] clarifying comments, etc --- DataGateway.Config/RuntimeConfigPath.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 8dd8667a7c..ac9dfcf873 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -60,7 +60,7 @@ public class RuntimeConfigPath /// /// Parse Json and replace @env('ENVIRONMENT_VARIABLE_NAME') with - /// the environment variable that corresponds to ENVIRONMENT_VARIABLE_NAME. + /// the environment variable's value that corresponds to ENVIRONMENT_VARIABLE_NAME. /// If no environment variable is found with that name, throw exception. /// /// Json string representing the runtime config file. @@ -68,6 +68,8 @@ public class RuntimeConfigPath public static string? ParseConfigJsonAndReplaceEnvVariables(string json) { StringBuilder stringBuilder = new(); + // string writer will modify string builder allowing + // us to return the string builder toString(). StringWriter stringWriter = new(stringBuilder); using JsonTextReader reader = new(new StringReader(json)); using JsonTextWriter writer = new(stringWriter) @@ -75,7 +77,7 @@ public class RuntimeConfigPath Formatting = Formatting.Indented }; - // @env : match @env(' + // @env\(' : match @env(' // .*? : lazy match any character except newline 0 or more times // (?='\)) : look ahead for ') which will combine with our lazy match // ie: in @env('hello')goodbye') we match @env('hello') @@ -98,7 +100,7 @@ public class RuntimeConfigPath writer.WritePropertyName(reader.Value.ToString()!); break; case JsonToken.String: - string valueToWrite = Regex.Replace(reader.Value.ToString()!, envPattern, new MatchEvaluator(ReplaceWithEnvVariable)); + string valueToWrite = Regex.Replace(reader.Value.ToString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); writer.WriteValue(valueToWrite); break; default: @@ -122,6 +124,7 @@ public class RuntimeConfigPath case JsonToken.EndObject: writer.WriteEndObject(); break; + // ie: "path" : null case JsonToken.Null: writer.WriteNull(); break; @@ -140,7 +143,7 @@ public class RuntimeConfigPath /// The match holding the environment variable name. /// The environment variable associated with the provided name. /// - private static string ReplaceWithEnvVariable(Match match) + private static string ReplaceMatchWithEnvVariable(Match match) { // [^@env\(] : any substring that is not @env( // .* : any char except newline any number of times From c82261c0cc8e15f6b932d9b6db604fc3aa01939a Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 12:10:47 -0700 Subject: [PATCH 08/16] clarifying comments --- DataGateway.Config/RuntimeConfigPath.cs | 4 ++-- .../Unittests/RuntimeConfigPathUnitTests.cs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index ac9dfcf873..ec60cff912 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -137,11 +137,11 @@ public class RuntimeConfigPath /// /// Retrieves the name of the environment variable - /// and then returns the environment variable associated + /// and then returns the environment variable value associated /// with that name, throwing an exception if none is found. /// /// The match holding the environment variable name. - /// The environment variable associated with the provided name. + /// The environment variable value associated with the provided name. /// private static string ReplaceMatchWithEnvVariable(Match match) { diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index 2304361e58..8ec928c487 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -45,6 +45,11 @@ public class RuntimeConfigPathUnitTests new string[] { "envVarValue", "@env(envVarValue)", "@enenvVarValue", "@env'()@env'envVarValue')')" } }, DisplayName = "Replacement strings that match.")] + // the following are envionrment variable names set to the + // associated values: + // 'envVarName -> _envVarName + // envVarName' -> envVarName_ + // 'envVarName' -> _envVarName_ [DataRow(new string[] { "@env(')", "@env()", "@env('envVarName')", "@env(''envVarName')", "@env('envVarName'')", "@env(''envVarName'')" }, new object[] { From ddea198ccce8031823947bd14ed024d1389e11b4 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 12:22:56 -0700 Subject: [PATCH 09/16] added clarifying comment --- DataGateway.Config/RuntimeConfigPath.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index ec60cff912..059e817c05 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -84,6 +84,11 @@ public class RuntimeConfigPath // '\) : consume the ') into the match (look ahead doesn't capture) // This pattern lazy matches any string that starts with @env(' and ends with ') // ie: fooBAR@env('hello-world')bash)FOO') match: @env('hello-world') + // This matching pattern allows for the @env('') to be safely nested + // within strings that contain ') after our match. + // ie: if the environment variable "Baz" has the value of "Bar" + // fooBarBaz: "('foo@env('Baz')Baz')" would parse into + // fooBarBaz: "('fooBarBaz')" string envPattern = @"@env\('.*?(?='\))'\)"; // The approach for parsing is to re-write the Json to a new string From 334adf16d5e68e7a7813b7c9df29ede3a627c37e Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 18:43:59 -0700 Subject: [PATCH 10/16] convert newtonsoft to system.text.json --- DataGateway.Config/RuntimeConfigPath.cs | 64 ++++++++++++------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 059e817c05..0f6458a719 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -1,4 +1,6 @@ +using System.Buffers; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using Azure.DataGateway.Service.Exceptions; using Newtonsoft.Json; @@ -67,15 +69,11 @@ public class RuntimeConfigPath /// Parsed json string. public static string? ParseConfigJsonAndReplaceEnvVariables(string json) { - StringBuilder stringBuilder = new(); - // string writer will modify string builder allowing - // us to return the string builder toString(). - StringWriter stringWriter = new(stringBuilder); - using JsonTextReader reader = new(new StringReader(json)); - using JsonTextWriter writer = new(stringWriter) - { - Formatting = Formatting.Indented - }; + Utf8JsonReader reader = new(jsonData: Encoding.UTF8.GetBytes(json), + isFinalBlock: true, + state: new()); + MemoryStream stream = new(); + Utf8JsonWriter writer = new(stream, options: new(){ Indented = true }); // @env\(' : match @env(' // .*? : lazy match any character except newline 0 or more times @@ -89,6 +87,9 @@ public class RuntimeConfigPath // ie: if the environment variable "Baz" has the value of "Bar" // fooBarBaz: "('foo@env('Baz')Baz')" would parse into // fooBarBaz: "('fooBarBaz')" + // Note that there is no escape character currently for ') to exist + // within the name of the environment variable, but that ') is not + // a valid environment variable name in certain shells. string envPattern = @"@env\('.*?(?='\))'\)"; // The approach for parsing is to re-write the Json to a new string @@ -97,47 +98,46 @@ public class RuntimeConfigPath // getting the environment variable for replacement. while (reader.Read()) { - if (reader.Value is not null) - { switch (reader.TokenType) { - case JsonToken.PropertyName: - writer.WritePropertyName(reader.Value.ToString()!); + case JsonTokenType.PropertyName: + writer.WritePropertyName(reader.GetString()!); break; - case JsonToken.String: - string valueToWrite = Regex.Replace(reader.Value.ToString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); - writer.WriteValue(valueToWrite); + case JsonTokenType.String: + string valueToWrite = Regex.Replace(reader.GetString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); + writer.WriteStringValue(valueToWrite); break; - default: - writer.WriteValue(reader.Value); + case JsonTokenType.Number: + writer.WriteNumberValue(reader.GetDecimal()); break; - } - } - else - { - switch (reader.TokenType) - { - case JsonToken.StartObject: + case JsonTokenType.True: + case JsonTokenType.False: + writer.WriteBooleanValue(reader.GetBoolean()); + break; + case JsonTokenType.StartObject: writer.WriteStartObject(); break; - case JsonToken.StartArray: + case JsonTokenType.StartArray: writer.WriteStartArray(); break; - case JsonToken.EndArray: + case JsonTokenType.EndArray: writer.WriteEndArray(); break; - case JsonToken.EndObject: + case JsonTokenType.EndObject: writer.WriteEndObject(); break; // ie: "path" : null - case JsonToken.Null: - writer.WriteNull(); + case JsonTokenType.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(reader.GetString()!); break; - } } } - return stringBuilder.ToString(); + writer.Flush(); + return Encoding.UTF8.GetString(stream.ToArray()); } /// From a3ef2a98dd28bc0e88db86a43b31c185de8c3ed0 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 18:47:49 -0700 Subject: [PATCH 11/16] removed using no longer required --- DataGateway.Config/RuntimeConfigPath.cs | 2 -- .../Unittests/RuntimeConfigPathUnitTests.cs | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 0f6458a719..9b77456a2d 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -1,9 +1,7 @@ -using System.Buffers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using Azure.DataGateway.Service.Exceptions; -using Newtonsoft.Json; namespace Azure.DataGateway.Config { diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index 8ec928c487..a905ca697f 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -212,7 +212,7 @@ public static string GetModifiedJsonString(string[] reps) ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""websiteplacement"": { - ""cardinality"": ""one"", + ""cardinality"": 1, ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""reviews"": { @@ -261,7 +261,7 @@ public static string GetModifiedJsonString(string[] reps) ], ""relationships"": { ""books"": { - ""cardinality"": ""one"", + ""cardinality"": 1.1, ""target.entity"": """ + reps[++index % reps.Length] + @""" } } From 192edc5917f0f299980aef7f80f367cfe0872471 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 20:14:06 -0700 Subject: [PATCH 12/16] removed reference to newtonsoft, formatting --- .../Azure.DataGateway.Config.csproj | 1 - DataGateway.Config/RuntimeConfigPath.cs | 68 +++++++++---------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/DataGateway.Config/Azure.DataGateway.Config.csproj b/DataGateway.Config/Azure.DataGateway.Config.csproj index d046e3e7c8..e55bacd831 100644 --- a/DataGateway.Config/Azure.DataGateway.Config.csproj +++ b/DataGateway.Config/Azure.DataGateway.Config.csproj @@ -8,7 +8,6 @@ - diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 9b77456a2d..163cc1b11b 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -96,41 +96,41 @@ public class RuntimeConfigPath // getting the environment variable for replacement. while (reader.Read()) { - switch (reader.TokenType) - { - case JsonTokenType.PropertyName: - writer.WritePropertyName(reader.GetString()!); - break; - case JsonTokenType.String: - string valueToWrite = Regex.Replace(reader.GetString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); - writer.WriteStringValue(valueToWrite); - break; - case JsonTokenType.Number: - writer.WriteNumberValue(reader.GetDecimal()); - break; - case JsonTokenType.True: - case JsonTokenType.False: - writer.WriteBooleanValue(reader.GetBoolean()); - break; + switch (reader.TokenType) + { + case JsonTokenType.PropertyName: + writer.WritePropertyName(reader.GetString()!); + break; + case JsonTokenType.String: + string valueToWrite = Regex.Replace(reader.GetString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); + writer.WriteStringValue(valueToWrite); + break; + case JsonTokenType.Number: + writer.WriteNumberValue(reader.GetDecimal()); + break; + case JsonTokenType.True: + case JsonTokenType.False: + writer.WriteBooleanValue(reader.GetBoolean()); + break; case JsonTokenType.StartObject: - writer.WriteStartObject(); - break; - case JsonTokenType.StartArray: - writer.WriteStartArray(); - break; - case JsonTokenType.EndArray: - writer.WriteEndArray(); - break; - case JsonTokenType.EndObject: - writer.WriteEndObject(); - break; - // ie: "path" : null - case JsonTokenType.Null: - writer.WriteNullValue(); - break; - default: - writer.WriteRawValue(reader.GetString()!); - break; + writer.WriteStartObject(); + break; + case JsonTokenType.StartArray: + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + writer.WriteEndArray(); + break; + case JsonTokenType.EndObject: + writer.WriteEndObject(); + break; + // ie: "path" : null + case JsonTokenType.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(reader.GetString()!); + break; } } From 85d39edc4886a97b93d0640c078fb5c4d9c7ef00 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Tue, 21 Jun 2022 20:22:58 -0700 Subject: [PATCH 13/16] formatting --- DataGateway.Config/RuntimeConfigPath.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 163cc1b11b..2e75ebb86c 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -71,7 +71,7 @@ public class RuntimeConfigPath isFinalBlock: true, state: new()); MemoryStream stream = new(); - Utf8JsonWriter writer = new(stream, options: new(){ Indented = true }); + Utf8JsonWriter writer = new(stream, options: new() { Indented = true }); // @env\(' : match @env(' // .*? : lazy match any character except newline 0 or more times @@ -192,10 +192,10 @@ public static string GetFileNameForEnvironment(string? hostingEnvironmentName) index++) { if (!string.IsNullOrWhiteSpace(environmentPrecedence[index]) - // The last index is for the default case - the last fallback option - // where environmentPrecedence[index] is string.Empty - // for that case, we still need to get the file name considering overrides - // so need to do an OR on the last index here + // The last index is for the default case - the last fallback option + // where environmentPrecedence[index] is string.Empty + // for that case, we still need to get the file name considering overrides + // so need to do an OR on the last index here || index == environmentPrecedence.Length - 1) { configFileNameWithExtension = From 30e35289822ab1dfd749a168f511b592a83d8762 Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 23 Jun 2022 13:10:23 -0700 Subject: [PATCH 14/16] typo, more descriptive comment --- .../Unittests/RuntimeConfigPathUnitTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index a905ca697f..e64295537e 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -45,7 +45,8 @@ public class RuntimeConfigPathUnitTests new string[] { "envVarValue", "@env(envVarValue)", "@enenvVarValue", "@env'()@env'envVarValue')')" } }, DisplayName = "Replacement strings that match.")] - // the following are envionrment variable names set to the + // since we match strings surrounded by single quotes, + // the following are environment variable names set to the // associated values: // 'envVarName -> _envVarName // envVarName' -> envVarName_ From c9698a2812e9320e08b0832d77117c8a5b33c8ad Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 23 Jun 2022 13:13:58 -0700 Subject: [PATCH 15/16] added a clarifying comment --- .../Unittests/RuntimeConfigPathUnitTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index e64295537e..7739e334c6 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -80,6 +80,8 @@ public void CheckConfigEnvParsingTest(string[] repKeys, string[] repValues) [DataTestMethod] [DataRow("")] [DataRow("fooBARbaz")] + // extra single quote added to environment variable + // names to validate we don't match these [DataRow("''envVarName")] [DataRow("''envVarName'")] [DataRow("envVarName''")] From e542b1cbdd372a93a5e7ddc7638a06f5cbd5e76e Mon Sep 17 00:00:00 2001 From: aaron burtle Date: Thu, 23 Jun 2022 18:42:53 -0700 Subject: [PATCH 16/16] added new field to config for decimals --- .../Unittests/RuntimeConfigPathUnitTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs index 7739e334c6..4319d0c2b2 100644 --- a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -121,6 +121,10 @@ public static string GetModifiedJsonString(string[] reps) return @"{ ""$schema"": "".. /../project-hawaii/playground/hawaii.draft-01.schema.json"", + ""versioning"": { + ""version"": 1.1, + ""patch"": 1 + }, ""data-source"": { ""database-type"": """ + reps[index % reps.Length] + @""", ""connection-string"": ""server=datagateway;database=" + reps[++index % reps.Length] + @";uid=" + reps[++index % reps.Length] + @";Password=" + reps[++index % reps.Length] + @";Allow User Variables=true"", @@ -215,7 +219,7 @@ public static string GetModifiedJsonString(string[] reps) ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""websiteplacement"": { - ""cardinality"": 1, + ""cardinality"": ""one"", ""target.entity"": """ + reps[++index % reps.Length] + @""" }, ""reviews"": { @@ -264,7 +268,7 @@ public static string GetModifiedJsonString(string[] reps) ], ""relationships"": { ""books"": { - ""cardinality"": 1.1, + ""cardinality"": ""one"", ""target.entity"": """ + reps[++index % reps.Length] + @""" } }