Add template for minimal image and update tutorial (#7461)

images/linux/scripts/installers/apt-common.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 ################################################################################
-##  File:  basic.sh
+##  File:  apt-common.sh
 ##  Desc:  Installs basic command line utilities and dev packages
 ################################################################################
 source $HELPER_SCRIPTS/install.sh

images/linux/scripts/installers/apt-vital.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+################################################################################
+##  File:  apt-vital.sh
+##  Desc:  Installs vital command line utilities
+################################################################################
+source $HELPER_SCRIPTS/install.sh
+
+vital_packages=$(get_toolset_value .apt.vital_packages[])
+apt-get install -y --no-install-recommends $vital_packages

images/linux/scripts/installers/docker-moby.sh

@@ -29,23 +29,27 @@ systemctl is-enabled --quiet docker.service || systemctl enable docker.service
 sleep 10
 docker info
 
-# If credentials are provided, attempt to log into Docker Hub
-# with a paid account to avoid Docker Hub's rate limit.
-if [ "${DOCKERHUB_LOGIN}" ] && [ "${DOCKERHUB_PASSWORD}" ]; then
-  docker login --username "${DOCKERHUB_LOGIN}" --password "${DOCKERHUB_PASSWORD}"
+if [ "${DOCKERHUB_PULL_IMAGES:-yes}" -eq "yes" ]; then
+    # If credentials are provided, attempt to log into Docker Hub
+    # with a paid account to avoid Docker Hub's rate limit.
+    if [ "${DOCKERHUB_LOGIN}" ] && [ "${DOCKERHUB_PASSWORD}" ]; then
+    docker login --username "${DOCKERHUB_LOGIN}" --password "${DOCKERHUB_PASSWORD}"
+    fi
+
+    # Pull images
+    images=$(get_toolset_value '.docker.images[]')
+    for image in $images; do
+        docker pull "$image"
+    done
+
+    # Always attempt to logout so we do not leave our credentials on the built
+    # image. Logout _should_ return a zero exit code even if no credentials were
+    # stored from earlier.
+    docker logout
+else
+    echo "Skipping docker images pulling"
 fi
 
-# Pull images
-images=$(get_toolset_value '.docker.images[]')
-for image in $images; do
-    docker pull "$image"
-done
-
-# Always attempt to logout so we do not leave our credentials on the built
-# image. Logout _should_ return a zero exit code even if no credentials were
-# stored from earlier.
-docker logout
-
 # Install amazon-ecr-credential-helper
 aws_latest_release_url="https://api.github.com/repos/awslabs/amazon-ecr-credential-helper/releases/latest"
 aws_helper_url=$(curl "${authString[@]}" -sL $aws_latest_release_url | jq -r '.body' | awk -F'[()]' '/linux-amd64/ {print $2}')
@@ -53,3 +57,6 @@ download_with_retries "$aws_helper_url" "/usr/bin" docker-credential-ecr-login
 chmod +x /usr/bin/docker-credential-ecr-login
 
 invoke_tests "Tools" "Docker"
+if [ "${DOCKERHUB_PULL_IMAGES:-yes}" -eq "yes" ]; then
+    invoke_tests "Tools" "Docker images"
+fi

images/linux/scripts/installers/post-deployment.sh

@@ -24,5 +24,11 @@ echo "PATH=$ENVPATH" | sudo tee -a /etc/environment
 echo "Updated /etc/environment: $(cat /etc/environment)"
 
 # Clean yarn and npm cache
-yarn cache clean
-npm cache clean --force
+if yarn --version > /dev/null
+then
+  yarn cache clean
+fi
+if npm --version
+then
+  npm cache clean --force
+fi

images/linux/scripts/tests/Apt.Tests.ps1

@@ -1,6 +1,6 @@
 Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1"
 
-$cmd = (Get-ToolsetContent).apt.cmd_packages
+$cmd = (Get-ToolsetContent).apt.cmd_packages + (Get-ToolsetContent).apt.vital_packages
 
 Describe "Apt" {
 

images/linux/scripts/tests/Tools.Tests.ps1

@@ -79,13 +79,13 @@ Describe "Docker" {
     It "docker-credential-ecr-login" {
         "docker-credential-ecr-login -v" | Should -ReturnZeroExitCode
     }
+}
 
-    Context "docker images" {
-        $testCases = (Get-ToolsetContent).docker.images | ForEach-Object { @{ ImageName = $_ } }
+Describe "Docker images" {
+    $testCases = (Get-ToolsetContent).docker.images | ForEach-Object { @{ ImageName = $_ } }
 
-        It "<ImageName>" -TestCases $testCases {
-           sudo docker images "$ImageName" --format "{{.Repository}}" | Should -Not -BeNullOrEmpty
-        }
+    It "<ImageName>" -TestCases $testCases {
+       sudo docker images "$ImageName" --format "{{.Repository}}" | Should -Not -BeNullOrEmpty
     }
 }
 

images/linux/toolsets/toolset-2004.json

@@ -133,13 +133,24 @@
         }
     ],
     "apt": {
+        "vital_packages": [
+            "bzip2",
+            "curl",
+            "g++",
+            "gcc",
+            "make",
+            "jq",
+            "tar",
+            "unzip",
+            "wget"
+        ],
         "common_packages": [
             "autoconf",
             "automake",
-            "build-essential",
             "dbus",
             "dnsutils",
             "dpkg",
+            "dpkg-dev",
             "fakeroot",
             "fonts-noto-color-emoji",
             "gnupg2",
@@ -149,6 +160,7 @@
             "lib32z1",
             "libc++abi-dev",
             "libc++-dev",
+            "libc6-dev",
             "libcurl4",
             "libgbm-dev",
             "libgconf-2-4",
@@ -171,7 +183,6 @@
             "pkg-config",
             "python-is-python3",
             "rpm",
-            "tar",
             "texinfo",
             "tk",
             "tzdata",
@@ -187,14 +198,11 @@
             "binutils",
             "bison",
             "brotli",
-            "bzip2",
             "coreutils",
-            "curl",
             "file",
             "flex",
             "ftp",
             "haveged",
-            "jq",
             "m4",
             "mediainfo",
             "netcat",
@@ -215,8 +223,6 @@
             "swig",
             "telnet",
             "time",
-            "unzip",
-            "wget",
             "zip"
         ]
     },

images/linux/toolsets/toolset-2204.json

@@ -116,13 +116,24 @@
         }
     ],
     "apt": {
+        "vital_packages": [
+            "bzip2",
+            "curl",
+            "g++",
+            "gcc",
+            "make",
+            "jq",
+            "tar",
+            "unzip",
+            "wget"
+        ],
         "common_packages": [
             "autoconf",
             "automake",
-            "build-essential",
             "dbus",
             "dnsutils",
             "dpkg",
+            "dpkg-dev",
             "fakeroot",
             "fonts-noto-color-emoji",
             "gnupg2",
@@ -132,6 +143,7 @@
             "lib32z1",
             "libc++abi-dev",
             "libc++-dev",
+            "libc6-dev",
             "libcurl4",
             "libgbm-dev",
             "libgconf-2-4",
@@ -155,7 +167,6 @@
             "pkg-config",
             "python-is-python3",
             "rpm",
-            "tar",
             "texinfo",
             "tk",
             "tzdata",
@@ -171,14 +182,11 @@
             "binutils",
             "bison",
             "brotli",
-            "bzip2",
             "coreutils",
-            "curl",
             "file",
             "flex",
             "ftp",
             "haveged",
-            "jq",
             "lz4",
             "m4",
             "mediainfo",
@@ -200,8 +208,6 @@
             "swig",
             "telnet",
             "time",
-            "unzip",
-            "wget",
             "zip"
         ]
     },

images/linux/ubuntu2004.json

@@ -153,6 +153,18 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts": [
+                "{{template_dir}}/scripts/installers/apt-vital.sh"
+            ],
+            "environment_vars": [
+                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "DEBIAN_FRONTEND=noninteractive"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts": [
@@ -193,10 +205,10 @@
         {
             "type": "shell",
             "scripts": [
+                "{{template_dir}}/scripts/installers/apt-common.sh",
                 "{{template_dir}}/scripts/installers/azcopy.sh",
                 "{{template_dir}}/scripts/installers/azure-cli.sh",
                 "{{template_dir}}/scripts/installers/azure-devops-cli.sh",
-                "{{template_dir}}/scripts/installers/basic.sh",
                 "{{template_dir}}/scripts/installers/bicep.sh",
                 "{{template_dir}}/scripts/installers/aliyun-cli.sh",
                 "{{template_dir}}/scripts/installers/apache.sh",

images/linux/ubuntu2204.pkr.hcl

@@ -256,6 +256,12 @@ build {
     scripts          = ["${path.root}/scripts/installers/configure-environment.sh"]
   }
 
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive", "HELPER_SCRIPTS=${local.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${local.installer_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/apt-vital.sh"]
+  }
+
   provisioner "shell" {
     environment_vars = ["HELPER_SCRIPTS=${var.helper_script_folder}"]
     execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
@@ -278,10 +284,10 @@ build {
     environment_vars = ["HELPER_SCRIPTS=${var.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${var.installer_script_folder}", "DEBIAN_FRONTEND=noninteractive"]
     execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
     scripts          = [
+                        "${path.root}/scripts/installers/apt-common.sh",
                         "${path.root}/scripts/installers/azcopy.sh",
                         "${path.root}/scripts/installers/azure-cli.sh",
                         "${path.root}/scripts/installers/azure-devops-cli.sh",
-                        "${path.root}/scripts/installers/basic.sh",
                         "${path.root}/scripts/installers/bicep.sh",
                         "${path.root}/scripts/installers/aliyun-cli.sh",
                         "${path.root}/scripts/installers/apache.sh",

images/linux/ubuntuminimal.pkr.hcl

@@ -0,0 +1,304 @@
+locals {
+  image_os = "ubuntu22"
+
+  toolset_file_name = "toolset-2204.json"
+
+  image_folder            = "/imagegeneration"
+  helper_script_folder    = "/imagegeneration/helpers"
+  installer_script_folder = "/imagegeneration/installers"
+  imagedata_file          = "/imagegeneration/imagedata.json"
+}
+
+variable "allowed_inbound_ip_addresses" {
+  type    = list(string)
+  default = []
+}
+
+variable "azure_tags" {
+  type    = map(string)
+  default = {}
+}
+
+variable "build_resource_group_name" {
+  type    = string
+  default = "${env("BUILD_RESOURCE_GROUP_NAME")}"
+}
+
+variable "capture_name_prefix" {
+  type    = string
+  default = "packer"
+}
+
+variable "client_id" {
+  type    = string
+  default = "${env("ARM_CLIENT_ID")}"
+}
+
+variable "client_secret" {
+  type      = string
+  default   = "${env("ARM_CLIENT_SECRET")}"
+  sensitive = true
+}
+
+variable "client_cert_path" {
+  type      = string
+  default   = "${env("ARM_CLIENT_CERT_PATH")}"
+}
+
+variable "commit_url" {
+  type      = string
+  default   = ""
+}
+
+variable "image_version" {
+  type    = string
+  default = "dev"
+}
+
+variable "install_password" {
+  type  = string
+  default = ""
+}
+
+variable "location" {
+  type    = string
+  default = "${env("ARM_RESOURCE_LOCATION")}"
+}
+
+variable "private_virtual_network_with_public_ip" {
+  type    = bool
+  default = false
+}
+
+variable "resource_group" {
+  type    = string
+  default = "${env("ARM_RESOURCE_GROUP")}"
+}
+
+variable "run_validation_diskspace" {
+  type    = bool
+  default = false
+}
+
+variable "storage_account" {
+  type    = string
+  default = "${env("ARM_STORAGE_ACCOUNT")}"
+}
+
+variable "subscription_id" {
+  type    = string
+  default = "${env("ARM_SUBSCRIPTION_ID")}"
+}
+
+variable "temp_resource_group_name" {
+  type    = string
+  default = "${env("TEMP_RESOURCE_GROUP_NAME")}"
+}
+
+variable "tenant_id" {
+  type    = string
+  default = "${env("ARM_TENANT_ID")}"
+}
+
+variable "virtual_network_name" {
+  type    = string
+  default = "${env("VNET_NAME")}"
+}
+
+variable "virtual_network_resource_group_name" {
+  type    = string
+  default = "${env("VNET_RESOURCE_GROUP")}"
+}
+
+variable "virtual_network_subnet_name" {
+  type    = string
+  default = "${env("VNET_SUBNET")}"
+}
+
+variable "vm_size" {
+  type    = string
+  default = "Standard_D4s_v4"
+}
+
+source "azure-arm" "build_vhd" {
+  location = "${var.location}"
+
+  // Auth
+  tenant_id        = "${var.tenant_id}"
+  subscription_id  = "${var.subscription_id}"
+  client_id        = "${var.client_id}"
+  client_secret    = "${var.client_secret}"
+  client_cert_path = "${var.client_cert_path}"
+
+  // Base image
+  image_offer     = "0001-com-ubuntu-server-jammy"
+  image_publisher = "canonical"
+  image_sku       = "22_04-lts"
+
+  // Target location
+  storage_account        = "${var.storage_account}"
+  resource_group_name    = "${var.resource_group}"
+  capture_container_name = "images"
+  capture_name_prefix    = "${var.capture_name_prefix}"
+
+  // Resource group for VM
+  build_resource_group_name = "${var.build_resource_group_name}"
+  temp_resource_group_name  = "${var.temp_resource_group_name}"
+
+  // Networking for VM
+  private_virtual_network_with_public_ip = "${var.private_virtual_network_with_public_ip}"
+  virtual_network_resource_group_name    = "${var.virtual_network_resource_group_name}"
+  virtual_network_name                   = "${var.virtual_network_name}"
+  virtual_network_subnet_name            = "${var.virtual_network_subnet_name}"
+  allowed_inbound_ip_addresses           = "${var.allowed_inbound_ip_addresses}"
+  
+  // VM Configuration
+  vm_size         = "${var.vm_size}"
+  os_disk_size_gb = "86"
+  os_type         = "Linux"
+  
+  dynamic "azure_tag" {
+    for_each = var.azure_tags
+    content {
+      name = azure_tag.key
+      value = azure_tag.value
+    }
+  }
+}
+
+build {
+  sources = ["source.azure-arm.build_vhd"]
+
+  // Create folder to store temporary data
+  provisioner "shell" {
+    execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    inline          = ["mkdir ${local.image_folder}",
+                       "chmod 777 ${local.image_folder}"]
+  }
+
+  // Add apt wrapper to implement retries
+  provisioner "shell" {
+    execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    script          = "${path.root}/scripts/base/apt-mock.sh"
+  }
+
+  // Install MS package repos
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/base/repos.sh"]
+  }
+
+  // Configure apt
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    script           = "${path.root}/scripts/base/apt.sh"
+  }
+
+  // Configure limits
+  provisioner "shell" {
+    execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    script          = "${path.root}/scripts/base/limits.sh"
+  }
+
+  provisioner "file" {
+    destination = "${local.helper_script_folder}"
+    source      = "${path.root}/scripts/helpers"
+  }
+
+  provisioner "file" {
+    destination = "${local.installer_script_folder}"
+    source      = "${path.root}/scripts/installers"
+  }
+
+  provisioner "file" {
+    destination = "${local.image_folder}"
+    sources     = [
+      "${path.root}/post-generation",
+      "${path.root}/scripts/tests"
+    ]
+  }
+
+  provisioner "file" {
+    destination = "${local.installer_script_folder}/toolset.json"
+    source      = "${path.root}/toolsets/${local.toolset_file_name}"
+  }
+
+  // Generate image data file
+  provisioner "shell" {
+    environment_vars = ["IMAGE_VERSION=${var.image_version}", "IMAGEDATA_FILE=${local.imagedata_file}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/preimagedata.sh"]
+  }
+
+  // Create /etc/environment, configure waagent etc.
+  provisioner "shell" {
+    environment_vars = ["IMAGE_VERSION=${var.image_version}", "IMAGE_OS=${local.image_os}", "HELPER_SCRIPTS=${local.helper_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/configure-environment.sh"]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive", "HELPER_SCRIPTS=${local.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${local.installer_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/apt-vital.sh"]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive", "HELPER_SCRIPTS=${local.helper_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/powershellcore.sh"]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["HELPER_SCRIPTS=${local.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${local.installer_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/Install-PowerShellModules.ps1"]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["DEBIAN_FRONTEND=noninteractive", "HELPER_SCRIPTS=${local.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${local.installer_script_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = [
+                        "${path.root}/scripts/installers/git.sh",
+                        "${path.root}/scripts/installers/github-cli.sh",
+                        "${path.root}/scripts/installers/zstd.sh"
+    ]
+  }
+
+  provisioner "shell" {
+    execute_command   = "/bin/sh -c '{{ .Vars }} {{ .Path }}'"
+    expect_disconnect = true
+    scripts           = ["${path.root}/scripts/base/reboot.sh"]
+  }
+
+  provisioner "shell" {
+    execute_command     = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    pause_before        = "1m0s"
+    scripts             = ["${path.root}/scripts/installers/cleanup.sh"]
+    start_retry_timeout = "10m"
+  }
+
+  provisioner "shell" {
+    execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    script          = "${path.root}/scripts/base/apt-mock-remove.sh"
+  }
+
+  provisioner "shell" {
+    environment_vars = ["HELPER_SCRIPT_FOLDER=${local.helper_script_folder}", "INSTALLER_SCRIPT_FOLDER=${local.installer_script_folder}", "IMAGE_FOLDER=${local.image_folder}"]
+    execute_command  = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    scripts          = ["${path.root}/scripts/installers/post-deployment.sh"]
+  }
+
+  provisioner "shell" {
+    environment_vars = ["RUN_VALIDATION=${var.run_validation_diskspace}"]
+    scripts          = ["${path.root}/scripts/installers/validate-disk-space.sh"]
+  }
+
+  provisioner "shell" {
+    execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+    inline          = ["sleep 30", "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync"]
+  }
+
+}