move apt pkgs to the toolset.json

images/linux/scripts/base/apt.sh

@@ -11,8 +11,10 @@ systemctl disable apt-daily-upgrade.service
 # Configure apt to always assume Y
 echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes
 
-# Use apt-fast for parallel downloads
+# Install aria2 and jq
 apt-get install aria2
+apt-get install jq
+# Use apt-fast for parallel downloads
 add-apt-repository -y ppa:apt-fast/stable
 apt-get update
-apt-get -y install apt-fast
+apt-get install apt-fast

images/linux/scripts/helpers/containercache.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-################################################################################
-##  File:  containercache.sh
-##  Desc:  Prepulls Docker images used in build tasks and templates
-################################################################################
-
-source $HELPER_SCRIPTS/document.sh
-
-# Check prereqs
-echo "Checking prereqs for image pulls"
-if ! command -v docker; then
-    echo "Docker is not installed, cant pull images"
-    exit 1
-fi
-
-# Information output
-systemctl status docker --no-pager
-
-# Pull images
-images=(
-    docker.io/jekyll/builder
-    mcr.microsoft.com/azure-pipelines/node8-typescript
-)
-
-for image in "${images[@]}"; do
-    docker pull "$image"
-done
-
-## Add container information to the metadata file
- DocumentInstalledItem "Cached container images"
-
-while read -r line; do
-    DocumentInstalledItemIndent "$line"
-done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')"

images/linux/scripts/installers/basic.sh

@@ -4,90 +4,13 @@
 ##  Desc:  Installs basic command line utilities and dev packages
 ################################################################################
 
+set -e
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/document.sh
-source $HELPER_SCRIPTS/os.sh
-
-set -e
-
-common_packages="dnsutils
-                 iproute2
-                 iputils-ping
-                 libunwind8
-                 locales
-                 openssh-client
-                 tzdata
-                 upx
-                 zstd
-                 libxkbfile-dev
-                 pkg-config
-                 libsecret-1-dev
-                 libxss1
-                 libgconf-2-4
-                 dbus
-                 xvfb
-                 libgbm-dev
-                 libgtk-3-0
-                 tk
-                 fakeroot
-                 dpkg
-                 rpm
-                 xz-utils
-                 xorriso
-                 zsync
-                 gnupg2
-                 lib32z1
-                 texinfo
-                 libsqlite3-dev
-                 libc++-dev
-                 libc++abi-dev"
-
-cmd_packages="curl
-              file
-              ftp
-              jq
-              netcat
-              ssh
-              parallel
-              rsync
-              shellcheck
-              sudo
-              telnet
-              time
-              unzip
-              zip
-              wget
-              m4
-              bison
-              flex
-              patchelf
-              bzip2
-              sqlite3
-              brotli
-              yamllint"
-
-if isUbuntu20 ; then
-    echo "Install python2"
-    apt-get install -y --no-install-recommends python-is-python2
-fi
-
-echo "Install libcurl"
-if isUbuntu16 || isUbuntu18; then
-   libcurelVer="libcurl3"
-fi
-
-if isUbuntu20 ; then
-    libcurelVer="libcurl4"
-fi
-
-apt-get install -y --no-install-recommends $libcurelVer
-
-# install additional packages only for Ubuntu16.04
-if isUbuntu16; then
-    common_packages="$common_packages
-            libicu55"
-fi
 
+toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+common_packages=$(jq -r ".apt.common_packages[]" $toolsetJson)
+cmd_packages=$(jq -r ".apt.cmd_packages[]" $toolsetJson)
 for package in $common_packages $cmd_packages; do
     echo "Install $package"
     apt-get install -y --no-install-recommends $package
@@ -108,5 +31,3 @@ DocumentInstalledItem "Basic packages:"
 for package in $common_packages $cmd_packages; do
     DocumentInstalledItemIndent $package
 done
-
-DocumentInstalledItemIndent "$libcurelVer"

images/linux/scripts/installers/docker-moby.sh

@@ -49,19 +49,28 @@ else
     set +e
 fi
 
-docker pull node:10
+# Pull images
-docker pull node:12
+images=(
-docker pull buildpack-deps:stretch
+    node:10
-docker pull buildpack-deps:buster
+    node:12
-docker pull node:10-alpine
+    buildpack-deps:stretch
-docker pull node:12-alpine
+    buildpack-deps:buster
-docker pull debian:8
+    node:10-alpine
-docker pull debian:9
+    node:12-alpine
-docker pull alpine:3.7
+    debian:8
-docker pull alpine:3.8
+    debian:9
-docker pull alpine:3.9
+    alpine:3.7
-docker pull alpine:3.10
+    alpine:3.8
-docker pull ubuntu:14.04
+    alpine:3.9
+    alpine:3.10
+    ubuntu:14.04
+    docker.io/jekyll/builder
+    mcr.microsoft.com/azure-pipelines/node8-typescript
+)
+
+for image in "${images[@]}"; do
+    docker pull "$image"
+done
 
 ## Add version information to the metadata file
 echo "Documenting Docker version"
@@ -71,3 +80,10 @@ DocumentInstalledItem "Docker-Moby ($docker_version)"
 echo "Documenting Docker-buildx version"
 DOCKER_BUILDX_VERSION=$(docker buildx version | cut -d ' ' -f2)
 DocumentInstalledItem "Docker-Buildx ($DOCKER_BUILDX_VERSION)"
+
+## Add container information to the metadata file
+DocumentInstalledItem "Cached container images"
+
+while read -r line; do
+    DocumentInstalledItemIndent "$line"
+done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')"

images/linux/toolsets/toolset-1604.json

@@ -108,5 +108,67 @@
                 "4.6.0"
             ]
         }
-    ]
+    ],
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl3",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libicu55",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
+        ]
+    }
 }

images/linux/toolsets/toolset-1804.json

@@ -104,5 +104,66 @@
                 "4.6.0"
             ]
         }
-    ]
+    ],
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl3",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
+        ]
+    }
 }

images/linux/toolsets/toolset-2004.json

@@ -67,5 +67,67 @@
             "ndk-bundle",
             "platform-tools"
         ]
+    },
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl4",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "python-is-python2",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
+        ]
     }
 }

images/linux/ubuntu1604.json

@@ -88,6 +88,16 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -198,31 +208,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1604.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-1604.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -232,7 +220,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu1804.json

@@ -91,6 +91,16 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -202,31 +212,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1804.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-1804.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -236,7 +224,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu2004.json

@@ -93,6 +93,16 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-2004.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-2004.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -204,31 +214,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-2004.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-2004.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -238,7 +226,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },