diff --git a/images/linux/scripts/base/apt.sh b/images/linux/scripts/base/apt.sh index b7ad9e7bb..8afcbe8d2 100644 --- a/images/linux/scripts/base/apt.sh +++ b/images/linux/scripts/base/apt.sh @@ -11,8 +11,10 @@ systemctl disable apt-daily-upgrade.service # Configure apt to always assume Y echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes -# Use apt-fast for parallel downloads +# Install aria2 and jq apt-get install aria2 +apt-get install jq +# Use apt-fast for parallel downloads add-apt-repository -y ppa:apt-fast/stable apt-get update -apt-get -y install apt-fast +apt-get install apt-fast diff --git a/images/linux/scripts/helpers/containercache.sh b/images/linux/scripts/helpers/containercache.sh deleted file mode 100644 index 018bcf534..000000000 --- a/images/linux/scripts/helpers/containercache.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -################################################################################ -## File: containercache.sh -## Desc: Prepulls Docker images used in build tasks and templates -################################################################################ - -source $HELPER_SCRIPTS/document.sh - -# Check prereqs -echo "Checking prereqs for image pulls" -if ! command -v docker; then - echo "Docker is not installed, cant pull images" - exit 1 -fi - -# Information output -systemctl status docker --no-pager - -# Pull images -images=( - docker.io/jekyll/builder - mcr.microsoft.com/azure-pipelines/node8-typescript -) - -for image in "${images[@]}"; do - docker pull "$image" -done - -## Add container information to the metadata file - DocumentInstalledItem "Cached container images" - -while read -r line; do - DocumentInstalledItemIndent "$line" -done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')" diff --git a/images/linux/scripts/installers/basic.sh b/images/linux/scripts/installers/basic.sh index 0a9e0e1b4..d216bffb3 100644 --- a/images/linux/scripts/installers/basic.sh +++ b/images/linux/scripts/installers/basic.sh @@ -4,90 +4,13 @@ ## Desc: Installs basic command line utilities and dev packages ################################################################################ +set -e # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -source $HELPER_SCRIPTS/os.sh - -set -e - -common_packages="dnsutils - iproute2 - iputils-ping - libunwind8 - locales - openssh-client - tzdata - upx - zstd - libxkbfile-dev - pkg-config - libsecret-1-dev - libxss1 - libgconf-2-4 - dbus - xvfb - libgbm-dev - libgtk-3-0 - tk - fakeroot - dpkg - rpm - xz-utils - xorriso - zsync - gnupg2 - lib32z1 - texinfo - libsqlite3-dev - libc++-dev - libc++abi-dev" - -cmd_packages="curl - file - ftp - jq - netcat - ssh - parallel - rsync - shellcheck - sudo - telnet - time - unzip - zip - wget - m4 - bison - flex - patchelf - bzip2 - sqlite3 - brotli - yamllint" - -if isUbuntu20 ; then - echo "Install python2" - apt-get install -y --no-install-recommends python-is-python2 -fi - -echo "Install libcurl" -if isUbuntu16 || isUbuntu18; then - libcurelVer="libcurl3" -fi - -if isUbuntu20 ; then - libcurelVer="libcurl4" -fi - -apt-get install -y --no-install-recommends $libcurelVer - -# install additional packages only for Ubuntu16.04 -if isUbuntu16; then - common_packages="$common_packages - libicu55" -fi +toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json" +common_packages=$(jq -r ".apt.common_packages[]" $toolsetJson) +cmd_packages=$(jq -r ".apt.cmd_packages[]" $toolsetJson) for package in $common_packages $cmd_packages; do echo "Install $package" apt-get install -y --no-install-recommends $package @@ -108,5 +31,3 @@ DocumentInstalledItem "Basic packages:" for package in $common_packages $cmd_packages; do DocumentInstalledItemIndent $package done - -DocumentInstalledItemIndent "$libcurelVer" diff --git a/images/linux/scripts/installers/docker-moby.sh b/images/linux/scripts/installers/docker-moby.sh index 7ab457de8..0ce51b7ec 100644 --- a/images/linux/scripts/installers/docker-moby.sh +++ b/images/linux/scripts/installers/docker-moby.sh @@ -49,19 +49,28 @@ else set +e fi -docker pull node:10 -docker pull node:12 -docker pull buildpack-deps:stretch -docker pull buildpack-deps:buster -docker pull node:10-alpine -docker pull node:12-alpine -docker pull debian:8 -docker pull debian:9 -docker pull alpine:3.7 -docker pull alpine:3.8 -docker pull alpine:3.9 -docker pull alpine:3.10 -docker pull ubuntu:14.04 +# Pull images +images=( + node:10 + node:12 + buildpack-deps:stretch + buildpack-deps:buster + node:10-alpine + node:12-alpine + debian:8 + debian:9 + alpine:3.7 + alpine:3.8 + alpine:3.9 + alpine:3.10 + ubuntu:14.04 + docker.io/jekyll/builder + mcr.microsoft.com/azure-pipelines/node8-typescript +) + +for image in "${images[@]}"; do + docker pull "$image" +done ## Add version information to the metadata file echo "Documenting Docker version" @@ -71,3 +80,10 @@ DocumentInstalledItem "Docker-Moby ($docker_version)" echo "Documenting Docker-buildx version" DOCKER_BUILDX_VERSION=$(docker buildx version | cut -d ' ' -f2) DocumentInstalledItem "Docker-Buildx ($DOCKER_BUILDX_VERSION)" + +## Add container information to the metadata file +DocumentInstalledItem "Cached container images" + +while read -r line; do + DocumentInstalledItemIndent "$line" +done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')" \ No newline at end of file diff --git a/images/linux/toolsets/toolset-1604.json b/images/linux/toolsets/toolset-1604.json index 40ea0af4b..f03740059 100644 --- a/images/linux/toolsets/toolset-1604.json +++ b/images/linux/toolsets/toolset-1604.json @@ -108,5 +108,67 @@ "4.6.0" ] } - ] + ], + "apt": { + "common_packages": [ + "dbus", + "dnsutils", + "dpkg", + "fakeroot", + "gnupg2", + "iproute2", + "iputils-ping", + "lib32z1", + "libc++abi-dev", + "libc++-dev", + "libcurl3", + "libgbm-dev", + "libgconf-2-4", + "libgtk-3-0", + "libicu55", + "libsecret-1-dev", + "libsqlite3-dev", + "libunwind8", + "libxkbfile-dev", + "libxss1", + "locales", + "openssh-client", + "pkg-config", + "rpm", + "texinfo", + "tk", + "tzdata", + "upx", + "xorriso", + "xvfb", + "xz-utils", + "zstd", + "zsync" + ], + "cmd_packages": [ + "bison", + "brotli", + "bzip2", + "curl", + "file", + "flex", + "ftp", + "jq", + "m4", + "netcat", + "parallel", + "patchelf", + "rsync", + "shellcheck", + "sqlite3", + "ssh", + "sudo", + "telnet", + "time", + "unzip", + "wget", + "yamllint", + "zip" + ] + } } diff --git a/images/linux/toolsets/toolset-1804.json b/images/linux/toolsets/toolset-1804.json index 1bd31b15d..f1ea16b56 100644 --- a/images/linux/toolsets/toolset-1804.json +++ b/images/linux/toolsets/toolset-1804.json @@ -104,5 +104,66 @@ "4.6.0" ] } - ] + ], + "apt": { + "common_packages": [ + "dbus", + "dnsutils", + "dpkg", + "fakeroot", + "gnupg2", + "iproute2", + "iputils-ping", + "lib32z1", + "libc++abi-dev", + "libc++-dev", + "libcurl3", + "libgbm-dev", + "libgconf-2-4", + "libgtk-3-0", + "libsecret-1-dev", + "libsqlite3-dev", + "libunwind8", + "libxkbfile-dev", + "libxss1", + "locales", + "openssh-client", + "pkg-config", + "rpm", + "texinfo", + "tk", + "tzdata", + "upx", + "xorriso", + "xvfb", + "xz-utils", + "zstd", + "zsync" + ], + "cmd_packages": [ + "bison", + "brotli", + "bzip2", + "curl", + "file", + "flex", + "ftp", + "jq", + "m4", + "netcat", + "parallel", + "patchelf", + "rsync", + "shellcheck", + "sqlite3", + "ssh", + "sudo", + "telnet", + "time", + "unzip", + "wget", + "yamllint", + "zip" + ] + } } diff --git a/images/linux/toolsets/toolset-2004.json b/images/linux/toolsets/toolset-2004.json index db07186fb..b4aa7ec41 100644 --- a/images/linux/toolsets/toolset-2004.json +++ b/images/linux/toolsets/toolset-2004.json @@ -67,5 +67,67 @@ "ndk-bundle", "platform-tools" ] + }, + "apt": { + "common_packages": [ + "dbus", + "dnsutils", + "dpkg", + "fakeroot", + "gnupg2", + "iproute2", + "iputils-ping", + "lib32z1", + "libc++abi-dev", + "libc++-dev", + "libcurl4", + "libgbm-dev", + "libgconf-2-4", + "libgtk-3-0", + "libsecret-1-dev", + "libsqlite3-dev", + "libunwind8", + "libxkbfile-dev", + "libxss1", + "locales", + "openssh-client", + "pkg-config", + "python-is-python2", + "rpm", + "texinfo", + "tk", + "tzdata", + "upx", + "xorriso", + "xvfb", + "xz-utils", + "zstd", + "zsync" + ], + "cmd_packages": [ + "bison", + "brotli", + "bzip2", + "curl", + "file", + "flex", + "ftp", + "jq", + "m4", + "netcat", + "parallel", + "patchelf", + "rsync", + "shellcheck", + "sqlite3", + "ssh", + "sudo", + "telnet", + "time", + "unzip", + "wget", + "yamllint", + "zip" + ] } } diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 3625b7ed2..c0d3b39d7 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -88,6 +88,16 @@ "source": "{{template_dir}}/scripts/installers", "destination": "{{user `installer_script_folder`}}" }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolcache-1604.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolset-1604.json", + "destination": "{{user `installer_script_folder`}}/toolset.json" + }, { "type": "shell", "scripts":[ @@ -198,31 +208,9 @@ "{{template_dir}}/scripts/installers/mongodb.sh", "{{template_dir}}/scripts/installers/rndgenerator.sh", "{{template_dir}}/scripts/installers/swig.sh", - "{{template_dir}}/scripts/installers/netlify.sh" - ], - "environment_vars": [ - "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}", - "DEBIAN_FRONTEND=noninteractive" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolcache-1604.json", - "destination": "{{user `installer_script_folder`}}/toolcache.json" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolset-1604.json", - "destination": "{{user `installer_script_folder`}}/toolset.json" - }, - { - "type": "shell", - "scripts":[ + "{{template_dir}}/scripts/installers/netlify.sh", "{{template_dir}}/scripts/installers/android.sh", "{{template_dir}}/scripts/installers/azpowershell.sh", - "{{template_dir}}/scripts/helpers/containercache.sh", "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/pypy.sh", "{{template_dir}}/scripts/installers/python.sh", @@ -232,7 +220,8 @@ "METADATA_FILE={{user `metadata_file`}}", "HELPER_SCRIPTS={{user `helper_script_folder`}}", "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", - "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" + "GITHUB_FEED_TOKEN={{user `github_feed_token`}}", + "DEBIAN_FRONTEND=noninteractive" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index b5037520d..95de0713d 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -91,6 +91,16 @@ "source": "{{template_dir}}/scripts/installers", "destination": "{{user `installer_script_folder`}}" }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolcache-1804.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolset-1804.json", + "destination": "{{user `installer_script_folder`}}/toolset.json" + }, { "type": "shell", "scripts":[ @@ -202,31 +212,9 @@ "{{template_dir}}/scripts/installers/mongodb.sh", "{{template_dir}}/scripts/installers/rndgenerator.sh", "{{template_dir}}/scripts/installers/swig.sh", - "{{template_dir}}/scripts/installers/netlify.sh" - ], - "environment_vars": [ - "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}", - "DEBIAN_FRONTEND=noninteractive" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolcache-1804.json", - "destination": "{{user `installer_script_folder`}}/toolcache.json" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolset-1804.json", - "destination": "{{user `installer_script_folder`}}/toolset.json" - }, - { - "type": "shell", - "scripts":[ + "{{template_dir}}/scripts/installers/netlify.sh", "{{template_dir}}/scripts/installers/android.sh", "{{template_dir}}/scripts/installers/azpowershell.sh", - "{{template_dir}}/scripts/helpers/containercache.sh", "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/pypy.sh", "{{template_dir}}/scripts/installers/python.sh", @@ -236,7 +224,8 @@ "METADATA_FILE={{user `metadata_file`}}", "HELPER_SCRIPTS={{user `helper_script_folder`}}", "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", - "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" + "GITHUB_FEED_TOKEN={{user `github_feed_token`}}", + "DEBIAN_FRONTEND=noninteractive" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 48d309b1e..8e2b69e49 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -93,6 +93,16 @@ "source": "{{template_dir}}/scripts/installers", "destination": "{{user `installer_script_folder`}}" }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolcache-2004.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, + { + "type": "file", + "source": "{{template_dir}}/toolsets/toolset-2004.json", + "destination": "{{user `installer_script_folder`}}/toolset.json" + }, { "type": "shell", "scripts":[ @@ -204,31 +214,9 @@ "{{template_dir}}/scripts/installers/mongodb.sh", "{{template_dir}}/scripts/installers/rndgenerator.sh", "{{template_dir}}/scripts/installers/swig.sh", - "{{template_dir}}/scripts/installers/netlify.sh" - ], - "environment_vars": [ - "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}", - "DEBIAN_FRONTEND=noninteractive" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolcache-2004.json", - "destination": "{{user `installer_script_folder`}}/toolcache.json" - }, - { - "type": "file", - "source": "{{template_dir}}/toolsets/toolset-2004.json", - "destination": "{{user `installer_script_folder`}}/toolset.json" - }, - { - "type": "shell", - "scripts":[ + "{{template_dir}}/scripts/installers/netlify.sh", "{{template_dir}}/scripts/installers/android.sh", "{{template_dir}}/scripts/installers/azpowershell.sh", - "{{template_dir}}/scripts/helpers/containercache.sh", "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/pypy.sh", "{{template_dir}}/scripts/installers/python.sh", @@ -238,7 +226,8 @@ "METADATA_FILE={{user `metadata_file`}}", "HELPER_SCRIPTS={{user `helper_script_folder`}}", "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", - "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" + "GITHUB_FEED_TOKEN={{user `github_feed_token`}}", + "DEBIAN_FRONTEND=noninteractive" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" },