diff --git a/DataGateway.Config/RuntimeConfigPath.cs b/DataGateway.Config/RuntimeConfigPath.cs index 9f6af9ee3b..2e75ebb86c 100644 --- a/DataGateway.Config/RuntimeConfigPath.cs +++ b/DataGateway.Config/RuntimeConfigPath.cs @@ -1,3 +1,8 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Azure.DataGateway.Service.Exceptions; + namespace Azure.DataGateway.Config { /// @@ -29,7 +34,7 @@ public class RuntimeConfigPath { if (File.Exists(ConfigFileName)) { - runtimeConfigJson = File.ReadAllText(ConfigFileName); + runtimeConfigJson = ParseConfigJsonAndReplaceEnvVariables(File.ReadAllText(ConfigFileName)); } else { @@ -53,6 +58,112 @@ public class RuntimeConfigPath return null; } + /// + /// Parse Json and replace @env('ENVIRONMENT_VARIABLE_NAME') with + /// the environment variable's value that corresponds to ENVIRONMENT_VARIABLE_NAME. + /// If no environment variable is found with that name, throw exception. + /// + /// Json string representing the runtime config file. + /// Parsed json string. + public static string? ParseConfigJsonAndReplaceEnvVariables(string json) + { + Utf8JsonReader reader = new(jsonData: Encoding.UTF8.GetBytes(json), + isFinalBlock: true, + state: new()); + MemoryStream stream = new(); + Utf8JsonWriter writer = new(stream, options: new() { Indented = true }); + + // @env\(' : match @env(' + // .*? : lazy match any character except newline 0 or more times + // (?='\)) : look ahead for ') which will combine with our lazy match + // ie: in @env('hello')goodbye') we match @env('hello') + // '\) : consume the ') into the match (look ahead doesn't capture) + // This pattern lazy matches any string that starts with @env(' and ends with ') + // ie: fooBAR@env('hello-world')bash)FOO') match: @env('hello-world') + // This matching pattern allows for the @env('') to be safely nested + // within strings that contain ') after our match. + // ie: if the environment variable "Baz" has the value of "Bar" + // fooBarBaz: "('foo@env('Baz')Baz')" would parse into + // fooBarBaz: "('fooBarBaz')" + // Note that there is no escape character currently for ') to exist + // within the name of the environment variable, but that ') is not + // a valid environment variable name in certain shells. + string envPattern = @"@env\('.*?(?='\))'\)"; + + // The approach for parsing is to re-write the Json to a new string + // as we read, using regex.replace for the matches we get from our + // pattern. We call a helper function for each match that handles + // getting the environment variable for replacement. + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonTokenType.PropertyName: + writer.WritePropertyName(reader.GetString()!); + break; + case JsonTokenType.String: + string valueToWrite = Regex.Replace(reader.GetString()!, envPattern, new MatchEvaluator(ReplaceMatchWithEnvVariable)); + writer.WriteStringValue(valueToWrite); + break; + case JsonTokenType.Number: + writer.WriteNumberValue(reader.GetDecimal()); + break; + case JsonTokenType.True: + case JsonTokenType.False: + writer.WriteBooleanValue(reader.GetBoolean()); + break; + case JsonTokenType.StartObject: + writer.WriteStartObject(); + break; + case JsonTokenType.StartArray: + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + writer.WriteEndArray(); + break; + case JsonTokenType.EndObject: + writer.WriteEndObject(); + break; + // ie: "path" : null + case JsonTokenType.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(reader.GetString()!); + break; + } + } + + writer.Flush(); + return Encoding.UTF8.GetString(stream.ToArray()); + } + + /// + /// Retrieves the name of the environment variable + /// and then returns the environment variable value associated + /// with that name, throwing an exception if none is found. + /// + /// The match holding the environment variable name. + /// The environment variable value associated with the provided name. + /// + private static string ReplaceMatchWithEnvVariable(Match match) + { + // [^@env\(] : any substring that is not @env( + // .* : any char except newline any number of times + // (?=\)) : look ahead for end char of ) + // This pattern greedy matches all characters that are not a part of @env() + // ie: @env('hello@env('goodbye')world') match: 'hello@env('goodbye')world' + string innerPattern = @"[^@env\(].*(?=\))"; + + // strip's first and last characters, ie: '''hello'' --> ''hello' + string envName = Regex.Match(match.Value, innerPattern).Value[1..^1]; + string? envValue = Environment.GetEnvironmentVariable(envName); + return envValue is not null ? envValue : + throw new DataGatewayException(message: $"Environmental Variable, {envName}, not found.", + statusCode: System.Net.HttpStatusCode.ServiceUnavailable, + subStatusCode: DataGatewayException.SubStatusCodes.ErrorInInitialization); + } + /// /// Precedence of environments is /// 1) Value of HAWAII_ENVIRONMENT. @@ -81,10 +192,10 @@ public static string GetFileNameForEnvironment(string? hostingEnvironmentName) index++) { if (!string.IsNullOrWhiteSpace(environmentPrecedence[index]) - // The last index is for the default case - the last fallback option - // where environmentPrecedence[index] is string.Empty - // for that case, we still need to get the file name considering overrides - // so need to do an OR on the last index here + // The last index is for the default case - the last fallback option + // where environmentPrecedence[index] is string.Empty + // for that case, we still need to get the file name considering overrides + // so need to do an OR on the last index here || index == environmentPrecedence.Length - 1) { configFileNameWithExtension = diff --git a/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs new file mode 100644 index 0000000000..4319d0c2b2 --- /dev/null +++ b/DataGateway.Service.Tests/Unittests/RuntimeConfigPathUnitTests.cs @@ -0,0 +1,448 @@ +using System; +using System.Data; +using Azure.DataGateway.Config; +using Azure.DataGateway.Service.Exceptions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; + +namespace Azure.DataGateway.Service.Tests.UnitTests +{ + /// + /// Unit tests for the environment variable + /// parser for the runtime configuration. These + /// tests verify that we parse the config correctly + /// when replacing environment variables. Also verify + /// we throw the right exception when environment + /// variable names are not found. + /// + [TestClass, TestCategory(TestCategory.MSSQL)] + public class RuntimeConfigPathUnitTests + { + #region Positive Tests + + /// + /// Test valid cases for parsing the runtime config. + /// These cases have strings close to the pattern we + /// match when looking to replace parts of the config, + /// strings that match said pattern, and other edge + /// cases to reveal if the pattern matching is working. + /// The pattern we look to match is @env('') where we take + /// what is inside of the '', ie: @env(''). The match is then + /// used to get the associated environment variable. + /// + /// Replacement used as key to get environment variable. + /// Replacement value. + [DataTestMethod] + [DataRow(new string[] { "@env(')", "@env()", "@env(')'@env('()", "@env('@env()'", "@@eennvv((''''))" }, + new object[] + { + new string[] { "@env(')", "@env()", "@env(')'@env('()", "@env('@env()'", "@@eennvv((''''))" } + }, + DisplayName = "Replacement strings that won't match.")] + [DataRow(new string[] { "@env('envVarName')", "@env(@env('envVarName'))", "@en@env('envVarName')", "@env'()@env'@env('envVarName')')')" }, + new object[] + { + new string[] { "envVarValue", "@env(envVarValue)", "@enenvVarValue", "@env'()@env'envVarValue')')" } + }, + DisplayName = "Replacement strings that match.")] + // since we match strings surrounded by single quotes, + // the following are environment variable names set to the + // associated values: + // 'envVarName -> _envVarName + // envVarName' -> envVarName_ + // 'envVarName' -> _envVarName_ + [DataRow(new string[] { "@env(')", "@env()", "@env('envVarName')", "@env(''envVarName')", "@env('envVarName'')", "@env(''envVarName'')" }, + new object[] + { + new string[] { "@env(')", "@env()", "envVarValue", "_envVarValue", "envVarValue_", "_envVarValue_" } + }, + DisplayName = "Replacement strings with some matches.")] + public void CheckConfigEnvParsingTest(string[] repKeys, string[] repValues) + { + SetEnvVariables(); + string expectedJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(GetModifiedJsonString(repValues)); + string actualJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(GetModifiedJsonString(repKeys)); + JObject expected = JObject.Parse(expectedJson); + JObject actual = JObject.Parse(actualJson); + Assert.IsTrue(JToken.DeepEquals(expected, actual)); + } + + #endregion Positive Tests + + #region Negative Tests + + /// + /// When we have a match that does not correspond + /// to a valid environment variable we throw an exception. + /// These tests verify this happens correctly. + /// + /// A match that is not a valid environment variable name. + [DataTestMethod] + [DataRow("")] + [DataRow("fooBARbaz")] + // extra single quote added to environment variable + // names to validate we don't match these + [DataRow("''envVarName")] + [DataRow("''envVarName'")] + [DataRow("envVarName''")] + [DataRow("''envVarName''")] + public void CheckConfigEnvParsingThrowExceptions(string invalidEnvVarName) + { + string json = @"{ ""foo"" : ""@env('envVarName'), @env('" + invalidEnvVarName + @"')"" }"; + SetEnvVariables(); + Assert.ThrowsException(() => RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(json)); + } + + #endregion Negative Tests + + #region Helper Functions + + /// + /// Setup some environment variables. + /// + private static void SetEnvVariables() + { + Environment.SetEnvironmentVariable($"envVarName", $"envVarValue"); + Environment.SetEnvironmentVariable($"'envVarName", $"_envVarValue"); + Environment.SetEnvironmentVariable($"envVarName'", $"envVarValue_"); + Environment.SetEnvironmentVariable($"'envVarName'", $"_envVarValue_"); + } + + /// + /// Modify the json string with the replacements provided. + /// This function cycles through the string array in a circular + /// fasion. + /// + /// Replacement strings. + /// Json string with replacements. + public static string GetModifiedJsonString(string[] reps) + { + int index = 0; + return +@"{ + ""$schema"": "".. /../project-hawaii/playground/hawaii.draft-01.schema.json"", + ""versioning"": { + ""version"": 1.1, + ""patch"": 1 + }, + ""data-source"": { + ""database-type"": """ + reps[index % reps.Length] + @""", + ""connection-string"": ""server=datagateway;database=" + reps[++index % reps.Length] + @";uid=" + reps[++index % reps.Length] + @";Password=" + reps[++index % reps.Length] + @";Allow User Variables=true"", + ""resolver-config-file"": """ + reps[++index % reps.Length] + @""" + }, + ""runtime"": { + ""rest"": { + ""enabled"": """ + reps[++index % reps.Length] + @""", + ""path"": ""/" + reps[++index % reps.Length] + @""" + }, + ""graphql"": { + ""enabled"": true, + ""path"": """ + reps[++index % reps.Length] + @""", + ""allow-introspection"": true + }, + ""host"": { + ""mode"": """ + reps[++index % reps.Length] + @""", + ""cors"": { + ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""" ], + ""allow-credentials"": """ + reps[++index % reps.Length] + @""" + }, + ""authentication"": { + ""provider"": """ + reps[++index % reps.Length] + @""", + ""jwt"": { + ""audience"": """", + ""issuer"": """", + ""issuer-key"": """ + reps[++index % reps.Length] + @""" + } + } + } + }, + ""entities"": { + ""Publisher"": { + ""source"": """ + reps[++index % reps.Length] + @"." + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Stock"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": null, + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", ""update"" ] + } + ], + ""relationships"": { + ""comics"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""", + ""source.fields"": [ ""categoryName"" ], + ""target.fields"": [ """ + reps[++index % reps.Length] + @""" ] + } + } + }, + ""Book"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""update"", """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + ""publishers"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""websiteplacement"": { + ""cardinality"": ""one"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""reviews"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + }, + ""authors"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": ""Author"", + ""linking.object"": ""book_author_link"", + ""linking.source.fields"": [ ""book_id"" ], + ""linking.target.fields"": [ """ + reps[++index % reps.Length] + @""" ] + } + }, + ""mappings"": { + ""id"": """ + reps[++index % reps.Length] + @""", + ""title"": """ + reps[++index % reps.Length] + @""" + } + }, + ""BookWebsitePlacement"": { + ""source"": ""book_website_placements"", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ + """ + reps[++index % reps.Length] + @""", + """ + reps[++index % reps.Length] + @""", + { + ""action"": ""delete"", + ""policy"": { + ""database"": ""@claims.id eq @item.id"" + }, + ""fields"": { + ""include"": [ ""*"" ], + ""exclude"": [ """ + reps[++index % reps.Length] + @""" ] + } + } + ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""one"", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Author"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": ""many"", + ""target.entity"": """ + reps[++index % reps.Length] + @""", + ""linking.object"": ""book_author_link"" + } + } + }, + ""Review"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + ""books"": { + ""cardinality"": """ + reps[++index % reps.Length] + @""", + ""target.entity"": """ + reps[++index % reps.Length] + @""" + } + } + }, + ""Comic"": { + ""source"": ""comics"", + ""rest"": true, + ""graphql"": null, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ null ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", """ + reps[++index % reps.Length] + @""" ] + } + ] + }, + ""Broker"": { + ""source"": ""brokers"", + ""graphql"": false, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ] + }, + ""WebsiteUser"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": false, + ""permissions"": [] + }, + ""SupportedType"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": false, + ""permissions"": [] + }, + ""stocks_price"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [] + }, + ""Tree"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": """ + reps[++index % reps.Length] + @""", + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""create"", """ + reps[++index % reps.Length] + @""", ""update"", ""delete"" ] + } + ], + ""mappings"": { + ""species"": ""Scientific Name"", + ""region"": ""United State's " + reps[++index % reps.Length] + @""" + } + }, + ""Shrub"": { + ""source"": ""trees"", + ""rest"": true, + ""permissions"": [ + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""create"", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] + } + ], + ""mappings"": { + ""species"": """ + reps[++index % reps.Length] + @""" + } + }, + ""Fungus"": { + ""source"": ""fungi"", + ""rest"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""", ""read"", """ + reps[++index % reps.Length] + @""", ""delete"" ] + } + ], + ""mappings"": { + ""spores"": ""hazards"" + } + }, + ""books_view_all"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + } + }, + ""stocks_view_selected"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ ""read"" ] + }, + { + ""role"": """ + reps[++index % reps.Length] + @""", + ""actions"": [ ""read"" ] + } + ], + ""relationships"": { + } + }, + ""books_publishers_view_composite"": { + ""source"": """ + reps[++index % reps.Length] + @""", + ""rest"": true, + ""graphql"": true, + ""permissions"": [ + { + ""role"": ""anonymous"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + }, + { + ""role"": ""authenticated"", + ""actions"": [ """ + reps[++index % reps.Length] + @""" ] + } + ], + ""relationships"": { + } + } + } +} +"; + } + + #endregion Helper Functions + } +}