From eee0743413c8c5d8ab9d9ea45e09a42ebeabb17d Mon Sep 17 00:00:00 2001 From: Mike Tesch Date: Fri, 12 Dec 2025 15:03:34 -0500 Subject: [PATCH] Add Ubuntu-Slim image definition (#13423) Add ubuntu-slim image definition --- .github/workflows/docker-images.yml | 31 +++ images/ubuntu-slim/Dockerfile | 60 +++++ .../ubuntu-slim/generate-software-report.sh | 81 ++++++ .../scripts/build/configure-apt-sources.sh | 19 ++ .../scripts/build/configure-apt.sh | 53 ++++ .../scripts/build/configure-dpkg.sh | 42 +++ .../scripts/build/configure-environment.sh | 72 ++++++ .../build/configure-image-data-file.sh | 40 +++ .../scripts/build/configure-system.sh | 20 ++ .../scripts/build/install-actions-cache.sh | 22 ++ .../scripts/build/install-apt-common.sh | 18 ++ .../scripts/build/install-apt-vital.sh | 12 + .../scripts/build/install-aws-tools.sh | 30 +++ .../scripts/build/install-azcopy.sh | 16 ++ .../scripts/build/install-azure-cli.sh | 13 + .../scripts/build/install-azure-devops-cli.sh | 16 ++ .../scripts/build/install-bicep.sh | 15 ++ .../scripts/build/install-git-lfs.sh | 20 ++ .../ubuntu-slim/scripts/build/install-git.sh | 34 +++ .../scripts/build/install-github-cli.sh | 22 ++ .../scripts/build/install-google-cloud-cli.sh | 20 ++ .../scripts/build/install-ms-repos.sh | 16 ++ .../scripts/build/install-nodejs.sh | 29 +++ .../ubuntu-slim/scripts/build/install-nvm.sh | 22 ++ .../scripts/build/install-pipx-packages.sh | 28 ++ .../scripts/build/install-powershell.sh | 15 ++ .../scripts/build/install-python.sh | 37 +++ .../ubuntu-slim/scripts/build/install-yq.sh | 22 ++ .../ubuntu-slim/scripts/build/install-zstd.sh | 36 +++ .../scripts/docs-gen/Common.Helpers.psm1 | 152 +++++++++++ .../docs-gen/Generate-SoftwareReport.ps1 | 75 ++++++ .../docs-gen/SoftwareReport.Common.psm1 | 81 ++++++ .../docs-gen/SoftwareReport.Helpers.psm1 | 37 +++ .../docs-gen/SoftwareReport.Tools.psm1 | 79 ++++++ images/ubuntu-slim/scripts/entrypoint.sh | 10 + images/ubuntu-slim/scripts/helpers/cleanup.sh | 12 + .../scripts/helpers/etc-environment.sh | 89 +++++++ images/ubuntu-slim/scripts/helpers/install.sh | 243 ++++++++++++++++++ images/ubuntu-slim/scripts/helpers/os.sh | 13 + images/ubuntu-slim/test.sh | 93 +++++++ images/ubuntu-slim/toolsets/toolset.json | 109 ++++++++ images/ubuntu-slim/ubuntu-slim-Readme.md | 118 +++++++++ .../ubuntu/scripts/helpers/etc-environment.sh | 4 +- 43 files changed, 1974 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker-images.yml create mode 100644 images/ubuntu-slim/Dockerfile create mode 100755 images/ubuntu-slim/generate-software-report.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-apt-sources.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-apt.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-dpkg.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-environment.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-image-data-file.sh create mode 100644 images/ubuntu-slim/scripts/build/configure-system.sh create mode 100644 images/ubuntu-slim/scripts/build/install-actions-cache.sh create mode 100644 images/ubuntu-slim/scripts/build/install-apt-common.sh create mode 100644 images/ubuntu-slim/scripts/build/install-apt-vital.sh create mode 100644 images/ubuntu-slim/scripts/build/install-aws-tools.sh create mode 100644 images/ubuntu-slim/scripts/build/install-azcopy.sh create mode 100644 images/ubuntu-slim/scripts/build/install-azure-cli.sh create mode 100644 images/ubuntu-slim/scripts/build/install-azure-devops-cli.sh create mode 100644 images/ubuntu-slim/scripts/build/install-bicep.sh create mode 100644 images/ubuntu-slim/scripts/build/install-git-lfs.sh create mode 100644 images/ubuntu-slim/scripts/build/install-git.sh create mode 100644 images/ubuntu-slim/scripts/build/install-github-cli.sh create mode 100644 images/ubuntu-slim/scripts/build/install-google-cloud-cli.sh create mode 100644 images/ubuntu-slim/scripts/build/install-ms-repos.sh create mode 100644 images/ubuntu-slim/scripts/build/install-nodejs.sh create mode 100644 images/ubuntu-slim/scripts/build/install-nvm.sh create mode 100644 images/ubuntu-slim/scripts/build/install-pipx-packages.sh create mode 100644 images/ubuntu-slim/scripts/build/install-powershell.sh create mode 100644 images/ubuntu-slim/scripts/build/install-python.sh create mode 100644 images/ubuntu-slim/scripts/build/install-yq.sh create mode 100644 images/ubuntu-slim/scripts/build/install-zstd.sh create mode 100644 images/ubuntu-slim/scripts/docs-gen/Common.Helpers.psm1 create mode 100644 images/ubuntu-slim/scripts/docs-gen/Generate-SoftwareReport.ps1 create mode 100644 images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Common.psm1 create mode 100644 images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Helpers.psm1 create mode 100644 images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Tools.psm1 create mode 100644 images/ubuntu-slim/scripts/entrypoint.sh create mode 100644 images/ubuntu-slim/scripts/helpers/cleanup.sh create mode 100644 images/ubuntu-slim/scripts/helpers/etc-environment.sh create mode 100644 images/ubuntu-slim/scripts/helpers/install.sh create mode 100644 images/ubuntu-slim/scripts/helpers/os.sh create mode 100755 images/ubuntu-slim/test.sh create mode 100644 images/ubuntu-slim/toolsets/toolset.json create mode 100644 images/ubuntu-slim/ubuntu-slim-Readme.md diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml new file mode 100644 index 000000000..6c1b5d733 --- /dev/null +++ b/.github/workflows/docker-images.yml @@ -0,0 +1,31 @@ +name: Test Docker Images + +on: + push: + branches: + - main + paths: + - 'images/ubuntu-slim/**' + - '.github/workflows/docker-images.yml' + pull_request: + paths: + - 'images/ubuntu-slim/**' + - '.github/workflows/docker-images.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + test-images: + runs-on: ubuntu-latest + strategy: + matrix: + directory: + - images/ubuntu-slim + steps: + - uses: actions/checkout@v6 + + - name: Run test.sh + working-directory: ${{ matrix.directory }} + run: ./test.sh diff --git a/images/ubuntu-slim/Dockerfile b/images/ubuntu-slim/Dockerfile new file mode 100644 index 000000000..043856e59 --- /dev/null +++ b/images/ubuntu-slim/Dockerfile @@ -0,0 +1,60 @@ +FROM ubuntu:24.04 AS base + +ARG IMAGE_VERSION=1.0.0 +ARG IMAGE_OWNER="GitHub" + +ENV IMAGE_OWNER=$IMAGE_OWNER +ENV ImageVersion=$IMAGE_VERSION +ENV IMAGE_VERSION=$IMAGE_VERSION +ENV ImageOS="Linux" +ENV IMAGE_TARGET_PLATFORM="GitHub" +ENV POWERSHELL_DISTRIBUTION_CHANNEL="GitHub-Actions-$ImageOS" +ENV IMAGEDATA_NAME="ubuntu:24.04" +ENV NVM_DIR="/etc/skel/.nvm" +ENV HELPER_SCRIPTS="/tmp/scripts/helpers" +ENV INSTALLER_SCRIPT_FOLDER="/tmp/toolsets" + +# Avoid interactive prompts +ENV DEBIAN_FRONTEND=noninteractive + +COPY scripts/build /tmp/scripts/build +COPY scripts/helpers /tmp/scripts/helpers +COPY toolsets/ /tmp/toolsets/ +RUN find /tmp/scripts -name "*.sh" -type f -exec chmod +x {} \; + +COPY scripts/entrypoint.sh /opt/entrypoint.sh +RUN chmod +x /opt/entrypoint.sh + +RUN apt-get update && apt-get upgrade -y && apt-get install -y sudo lsb-release jq dpkg && \ + touch /run/.containerenv && \ + /tmp/scripts/build/configure-apt.sh && \ + /tmp/scripts/build/configure-apt-sources.sh && \ + /tmp/scripts/build/install-apt-vital.sh && \ + /tmp/scripts/build/install-ms-repos.sh && \ + /tmp/scripts/build/configure-image-data-file.sh && \ + /tmp/scripts/build/configure-environment.sh && \ + /tmp/scripts/build/install-actions-cache.sh && \ + /tmp/scripts/build/install-apt-common.sh && \ + /tmp/scripts/build/install-azcopy.sh && \ + /tmp/scripts/build/install-azure-cli.sh && \ + /tmp/scripts/build/install-azure-devops-cli.sh && \ + /tmp/scripts/build/install-bicep.sh && \ + /tmp/scripts/build/install-aws-tools.sh && \ + /tmp/scripts/build/install-git.sh && \ + /tmp/scripts/build/install-git-lfs.sh && \ + /tmp/scripts/build/install-github-cli.sh && \ + /tmp/scripts/build/install-google-cloud-cli.sh && \ + /tmp/scripts/build/install-nvm.sh && \ + /tmp/scripts/build/install-nodejs.sh && \ + /tmp/scripts/build/install-powershell.sh && \ + /tmp/scripts/build/configure-dpkg.sh && \ + /tmp/scripts/build/install-yq.sh && \ + /tmp/scripts/build/install-python.sh && \ + /tmp/scripts/build/install-zstd.sh && \ + /tmp/scripts/build/install-pipx-packages.sh && \ + /tmp/scripts/build/configure-system.sh && \ + /tmp/scripts/helpers/cleanup.sh + +ENTRYPOINT ["/opt/entrypoint.sh"] + +CMD [ "bash" ] diff --git a/images/ubuntu-slim/generate-software-report.sh b/images/ubuntu-slim/generate-software-report.sh new file mode 100755 index 000000000..7696afdcd --- /dev/null +++ b/images/ubuntu-slim/generate-software-report.sh @@ -0,0 +1,81 @@ +#!/bin/bash -e + +# This script builds and runs various tests on the ubuntu-slim Docker image +# to ensure it contains the expected software and configurations. +# The build and test workflows for docker images expect this script to be present. +# +# Usage: test.sh [IMAGE_NAME] +# If IMAGE_NAME is not provided, defaults to ubuntu-slim:test + +show_help() { + echo "Usage: $0 [IMAGE_NAME]" + echo "" + echo "Generate a software report for a Docker image." + echo "" + echo "Arguments:" + echo " IMAGE_NAME Docker image name to generate report for (default: ubuntu-slim:test)" + echo "" + echo "Examples:" + echo " $0 # Generate report for ubuntu-slim:test (builds image first)" + echo " $0 my-registry/ubuntu:latest # Generate report for existing image" + echo " $0 ubuntu-slim:v1.2.3 # Generate report for tagged image" + echo "" + echo "Options:" + echo " -h, --help Show this help message" +} + +# Handle help flags +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + show_help + exit 0 +fi + +# Set the image name from parameter or use default +IMAGE_NAME="${1:-ubuntu-slim:test}" + +# Build the image only if using the default name (for backward compatibility) +if [[ "$IMAGE_NAME" == "ubuntu-slim:test" ]]; then + echo "Building image: $IMAGE_NAME" + docker build --debug --progress plain -t "$IMAGE_NAME" . +else + # Check if the image exists + if ! docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then + echo "Error: Image '$IMAGE_NAME' does not exist. Please build it first or provide a valid image name." + echo "Run '$0 --help' for usage information." + exit 1 + fi +fi + +echo "Generating software report for image: $IMAGE_NAME" + +# Get the script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$(cd ../../helpers/software-report-base && pwd)" + +echo $BASE_DIR + +# Create a temporary directory for output +OUTPUT_DIR=$(mktemp -d) +echo "Using temporary directory: $OUTPUT_DIR" + +# Run the container and execute the PowerShell script inside it +echo "Running Generate-SoftwareReport.ps1 inside the container..." +docker run --rm \ + -v "$OUTPUT_DIR:/output" \ + -v "$SCRIPT_DIR/scripts/docs-gen:/scripts/docs-gen:ro" \ + -v "$BASE_DIR:/scripts/software-report-base:ro" \ + "$IMAGE_NAME" \ + pwsh /scripts/docs-gen/Generate-SoftwareReport.ps1 -OutputDirectory /output + +if [ -f "$OUTPUT_DIR/software-report.md" ]; then + cp "$OUTPUT_DIR/software-report.md" ubuntu-slim-Readme.md + echo "✓ Copied software-report.md to current directory" +else + echo "✗ Error: software-report.md was not generated" + rm -rf "$OUTPUT_DIR" + exit 1 +fi + +# Clean up temporary directory +rm -rf "$OUTPUT_DIR" +echo "✓ Software report generation complete" diff --git a/images/ubuntu-slim/scripts/build/configure-apt-sources.sh b/images/ubuntu-slim/scripts/build/configure-apt-sources.sh new file mode 100644 index 000000000..d1125cf60 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-apt-sources.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e +################################################################################ +## File: configure-apt-sources.sh +## Desc: Configure apt sources with failover from Azure to Ubuntu archives. +################################################################################ + +source $HELPER_SCRIPTS/os.sh + +touch /etc/apt/apt-mirrors.txt + +printf "http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\n" | tee -a /etc/apt/apt-mirrors.txt +printf "https://archive.ubuntu.com/ubuntu/\tpriority:2\n" | tee -a /etc/apt/apt-mirrors.txt +printf "https://security.ubuntu.com/ubuntu/\tpriority:3\n" | tee -a /etc/apt/apt-mirrors.txt + +if is_ubuntu24; then + sed -i 's|http://azure\.archive\.ubuntu\.com/ubuntu/|mirror+file:/etc/apt/apt-mirrors.txt|' /etc/apt/sources.list.d/ubuntu.sources +else + sed -i 's|http://azure\.archive\.ubuntu\.com/ubuntu/|mirror+file:/etc/apt/apt-mirrors.txt|' /etc/apt/sources.list +fi diff --git a/images/ubuntu-slim/scripts/build/configure-apt.sh b/images/ubuntu-slim/scripts/build/configure-apt.sh new file mode 100644 index 000000000..5b9c56f5e --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-apt.sh @@ -0,0 +1,53 @@ +#!/bin/bash -e +################################################################################ +## File: configure-apt.sh +## Desc: Configure apt, install jq and apt-fast packages. +################################################################################ + +source $HELPER_SCRIPTS/os.sh + +# Stop and disable apt-daily upgrade services; +# systemctl stop apt-daily.timer +# systemctl disable apt-daily.timer +# systemctl disable apt-daily.service +# systemctl stop apt-daily-upgrade.timer +# systemctl disable apt-daily-upgrade.timer +# systemctl disable apt-daily-upgrade.service + +# Enable retry logic for apt up to 10 times +echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries + +# Configure apt to always assume Y +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes + +# APT understands a field called Phased-Update-Percentage which can be used to control the rollout of a new version. It is an integer between 0 and 100. +# In case you have multiple systems that you want to receive the same set of updates, +# you can set APT::Machine-ID to a UUID such that they all phase the same, +# or set APT::Get::Never-Include-Phased-Updates or APT::Get::Always-Include-Phased-Updates to true such that APT will never/always consider phased updates. +# apt-cache policy pkgname +echo 'APT::Get::Always-Include-Phased-Updates "true";' > /etc/apt/apt.conf.d/99-phased-updates + +# Fix bad proxy and http headers settings +cat <> /etc/apt/apt.conf.d/99bad_proxy +Acquire::http::Pipeline-Depth 0; +Acquire::http::No-Cache true; +Acquire::https::Pipeline-Depth 0; +Acquire::https::No-Cache true; +Acquire::BrokenProxy true; +EOF + +# Uninstall unattended-upgrades +apt-get purge unattended-upgrades + +echo 'APT sources' +if ! is_ubuntu24; then + cat /etc/apt/sources.list +else + cat /etc/apt/sources.list.d/ubuntu.sources +fi + +apt-get update +# Install jq +apt-get install jq + +echo "ubuntu ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/build/configure-dpkg.sh b/images/ubuntu-slim/scripts/build/configure-dpkg.sh new file mode 100644 index 000000000..bff62a932 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-dpkg.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e +################################################################################ +## File: configure-dpkg.sh +## Desc: Configure dpkg +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/etc-environment.sh +source $HELPER_SCRIPTS/os.sh +# This is the anti-frontend. It never interacts with you at all, +# and makes the default answers be used for all questions. It +# might mail error messages to root, but that's it; otherwise it +# is completely silent and unobtrusive, a perfect frontend for +# automatic installs. If you are using this front-end, and require +# non-default answers to questions, you will need to pre-seed the +# debconf database +set_etc_environment_variable "DEBIAN_FRONTEND" "noninteractive" + +# dpkg can be instructed not to ask for confirmation +# when replacing a configuration file (with the --force-confdef --force-confold options) +cat <> /etc/apt/apt.conf.d/10dpkg-options +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +} +EOF + +# hide information about packages that are no longer required +cat <> /etc/apt/apt.conf.d/10apt-autoremove +APT::Get::AutomaticRemove "0"; +APT::Get::HideAutoRemove "1"; +EOF + +# Install libicu70 package for Ubuntu 24 +if is_ubuntu24 ; then + wget https://archive.ubuntu.com/ubuntu/pool/main/i/icu/libicu70_70.1-2_amd64.deb + + EXPECTED_LIBICU_SHA512="a6315482d93606e375c272718d2458870b95e4ed4b672ea8640cf7bc2d2c2f41aea13b798b1e417e1ffc472a90c6aad150d3d293aa9bddec48e39106e4042807" + ACTUAL_LIBICU_SHA512="$(sha512sum "./libicu70_70.1-2_amd64.deb" | awk '{print $1}')" + [ "$EXPECTED_LIBICU_SHA512" = "$ACTUAL_LIBICU_SHA512" ] || { echo "libicu checksum mismatch in configure-dpkg.sh"; exit 1;} + sudo apt-get install -y ./libicu70_70.1-2_amd64.deb +fi diff --git a/images/ubuntu-slim/scripts/build/configure-environment.sh b/images/ubuntu-slim/scripts/build/configure-environment.sh new file mode 100644 index 000000000..806e9e930 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-environment.sh @@ -0,0 +1,72 @@ +#!/bin/bash -e +################################################################################ +## File: configure-environment.sh +## Desc: Configure system and environment +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/os.sh +source $HELPER_SCRIPTS/etc-environment.sh + +whoami + +# Set ImageVersion and ImageOS env variables +set_etc_environment_variable "ImageVersion" "${IMAGE_VERSION}" +set_etc_environment_variable "ImageOS" "${IMAGE_OS}" + +# Set the ACCEPT_EULA variable to Y value to confirm your acceptance of the End-User Licensing Agreement +set_etc_environment_variable "ACCEPT_EULA" "Y" + +# This directory is supposed to be created in $HOME and owned by user(https://github.com/actions/runner-images/issues/491) +mkdir -p /etc/skel/.config/configstore +set_etc_environment_variable "XDG_CONFIG_HOME" '$HOME/.config' + +# Prepare directory and env variable for toolcache +echo "Setting up AGENT_TOOLSDIRECTORY and RUNNER_TOOL_CACHE variable to /opt/hostedtoolcache" +AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache +mkdir $AGENT_TOOLSDIRECTORY +set_etc_environment_variable "AGENT_TOOLSDIRECTORY" "${AGENT_TOOLSDIRECTORY}" +set_etc_environment_variable "RUNNER_TOOL_CACHE" "${AGENT_TOOLSDIRECTORY}" +chmod -R 777 $AGENT_TOOLSDIRECTORY + +# https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html +# https://www.suse.com/support/kb/doc/?id=000016692 +echo 'vm.max_map_count=262144' | tee -a /etc/sysctl.conf + +# https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files +echo 'fs.inotify.max_user_watches=655360' | tee -a /etc/sysctl.conf +echo 'fs.inotify.max_user_instances=1280' | tee -a /etc/sysctl.conf + +# https://github.com/actions/runner-images/issues/9491 +echo 'vm.mmap_rnd_bits=28' | tee -a /etc/sysctl.conf + +# https://github.com/actions/runner-images/pull/7860 +netfilter_rule='/etc/udev/rules.d/50-netfilter.rules' +rules_directory="$(dirname "${netfilter_rule}")" +mkdir -p $rules_directory +touch $netfilter_rule +echo 'ACTION=="add", SUBSYSTEM=="module", KERNEL=="nf_conntrack", RUN+="/usr/sbin/sysctl net.netfilter.nf_conntrack_tcp_be_liberal=1"' | tee -a $netfilter_rule + +# Remove fwupd if installed. We're running on VMs in Azure and the fwupd package is not needed. +# Leaving it enable means periodic refreshes show in network traffic and firewall logs +# Check if fwupd-refresh.timer exists in systemd +if systemctl list-unit-files fwupd-refresh.timer &>/dev/null; then + echo "Masking fwupd-refresh.timer..." + systemctl mask fwupd-refresh.timer +fi + +# This is a legacy check, leaving for earlier versions of Ubuntu +# If fwupd config still exists, disable the motd updates +if [[ -f "/etc/fwupd/daemon.conf" ]]; then + sed -i 's/UpdateMotd=true/UpdateMotd=false/g' /etc/fwupd/daemon.conf +fi + +# Disable to load providers +# https://github.com/microsoft/azure-pipelines-agent/issues/3834 +if is_ubuntu22; then + sed -i 's/openssl_conf = openssl_init/#openssl_conf = openssl_init/g' /etc/ssl/openssl.cnf +fi + +# # Disable man-db auto update +# echo "set man-db/auto-update false" | debconf-communicate +# dpkg-reconfigure man-db diff --git a/images/ubuntu-slim/scripts/build/configure-image-data-file.sh b/images/ubuntu-slim/scripts/build/configure-image-data-file.sh new file mode 100644 index 000000000..10b5a93a5 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-image-data-file.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +function create_imagedata_json() { + + arch=$(uname -m) + if [[ $arch == "x86_64" ]]; then + arch="x64" + elif [[ $arch == "aarch64" ]]; then + arch="arm64" + else + echo "Unsupported architecture: $arch" + exit 1 + fi + + if [[ -n "$IMAGEDATA_INCLUDED_SOFTWARE" ]]; then + included_software="- Included Software: ${IMAGEDATA_INCLUDED_SOFTWARE}" + fi + + imagedata_file="/imagegeneration/imagedata.json" + + cat < $imagedata_file +[ + { + "group": "VM Image", + "detail": "- OS: Linux (${arch})\n- Source: Docker\n- Name: ${IMAGEDATA_NAME}\n- Version: ${IMAGE_VERSION}\n${included_software}" + } +] +EOF + +} + +mkdir -p /imagegeneration + +# Generate the imagedata JSON file displayed on workflow initialization +if [[ -n "$IMAGEDATA_NAME" ]]; then + echo "Generating imagedata JSON file" + create_imagedata_json +else + echo "IMAGEDATA_NAME is null or empty. Skipping imagedata JSON generation." +fi \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/build/configure-system.sh b/images/ubuntu-slim/scripts/build/configure-system.sh new file mode 100644 index 000000000..635942141 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/configure-system.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +################################################################################ +## File: configure-system.sh +## Desc: Post deployment system configuration actions +################################################################################ + +source $HELPER_SCRIPTS/etc-environment.sh +source $HELPER_SCRIPTS/os.sh + +echo "chmod -R 777 /opt" +chmod -R 777 /opt +echo "chmod -R 777 /usr/share" +chmod -R 777 /usr/share + +# Remove quotes around PATH +ENVPATH=$(grep 'PATH=' /etc/environment | head -n 1 | sed -z 's/^PATH=*//') +ENVPATH=${ENVPATH#"\""} +ENVPATH=${ENVPATH%"\""} +replace_etc_environment_variable "PATH" "${ENVPATH}" +echo "Updated /etc/environment: $(cat /etc/environment)" diff --git a/images/ubuntu-slim/scripts/build/install-actions-cache.sh b/images/ubuntu-slim/scripts/build/install-actions-cache.sh new file mode 100644 index 000000000..6a3132e12 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-actions-cache.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +################################################################################ +## File: install-actions-cache.sh +## Desc: Download latest release from https://github.com/actions/action-versions +## Maintainer: #actions-runtime and @TingluoHuang +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh +source $HELPER_SCRIPTS/etc-environment.sh + +# Prepare directory and env variable for ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE +ACTION_ARCHIVE_CACHE_DIR=/opt/actionarchivecache +mkdir -p $ACTION_ARCHIVE_CACHE_DIR +chmod -R 777 $ACTION_ARCHIVE_CACHE_DIR +echo "Setting up ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE variable to ${ACTION_ARCHIVE_CACHE_DIR}" +set_etc_environment_variable "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" "${ACTION_ARCHIVE_CACHE_DIR}" + +# Download latest release from github.com/actions/action-versions and untar to /opt/actionarchivecache +download_url=$(resolve_github_release_asset_url "actions/action-versions" "endswith(\"action-versions.tar.gz\")" "latest") +archive_path=$(download_with_retry "$download_url") +tar -xzf "$archive_path" -C $ACTION_ARCHIVE_CACHE_DIR diff --git a/images/ubuntu-slim/scripts/build/install-apt-common.sh b/images/ubuntu-slim/scripts/build/install-apt-common.sh new file mode 100644 index 000000000..a6fda8027 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-apt-common.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +################################################################################ +## File: install-apt-common.sh +## Desc: Install basic command line utilities and dev packages +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +common_packages=$(get_toolset_value .apt.common_packages[]) +cmd_packages=$(get_toolset_value .apt.cmd_packages[]) + +apt-get install --no-install-recommends $common_packages $cmd_packages + +# for package in $common_packages $cmd_packages; do +# echo "Install $package" +# apt-get install --no-install-recommends $package +# done diff --git a/images/ubuntu-slim/scripts/build/install-apt-vital.sh b/images/ubuntu-slim/scripts/build/install-apt-vital.sh new file mode 100644 index 000000000..305d4336b --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-apt-vital.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e +################################################################################ +## File: install-apt-vital.sh +## Desc: Install vital command line utilities +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +vital_packages=$(get_toolset_value .apt.vital_packages[]) +apt-get install --no-install-recommends $vital_packages + diff --git a/images/ubuntu-slim/scripts/build/install-aws-tools.sh b/images/ubuntu-slim/scripts/build/install-aws-tools.sh new file mode 100644 index 000000000..04ea661e9 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-aws-tools.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +################################################################################ +## File: install-aws-tools.sh +## Desc: Install the AWS CLI, Session Manager plugin for the AWS CLI, and AWS SAM CLI +## Supply chain security: AWS SAM CLI - checksum validation +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +awscliv2_archive_path=$(download_with_retry "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip") +unzip -qq "$awscliv2_archive_path" -d /tmp/installers/ +/tmp/installers/aws/install -i /usr/local/aws-cli -b /usr/local/bin + +smplugin_deb_path=$(download_with_retry "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb") +apt-get install "$smplugin_deb_path" + +# Download the latest aws sam cli release +aws_sam_cli_archive_name="aws-sam-cli-linux-x86_64.zip" +sam_cli_download_url=$(resolve_github_release_asset_url "aws/aws-sam-cli" "endswith(\"$aws_sam_cli_archive_name\")" "latest") +aws_sam_cli_archive_path=$(download_with_retry "$sam_cli_download_url") + +# Supply chain security - AWS SAM CLI +aws_sam_cli_hash=$(get_checksum_from_github_release "aws/aws-sam-cli" "${aws_sam_cli_archive_name}.. " "latest" "SHA256") +use_checksum_comparison "$aws_sam_cli_archive_path" "$aws_sam_cli_hash" + +# Install the latest aws sam cli release +mkdir -p /tmp/installers/aws-sam-cli +unzip "$aws_sam_cli_archive_path" -d /tmp/installers/aws-sam-cli +/tmp/installers/aws-sam-cli/install -i /usr/local/aws-sam-cli -b /usr/local/bin diff --git a/images/ubuntu-slim/scripts/build/install-azcopy.sh b/images/ubuntu-slim/scripts/build/install-azcopy.sh new file mode 100644 index 000000000..402b264ee --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-azcopy.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +################################################################################ +## File: install-azcopy.sh +## Desc: Install AzCopy +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Install AzCopy10 +archive_path=$(download_with_retry "https://aka.ms/downloadazcopy-v10-linux") +tar xzf "$archive_path" --strip-components=1 -C /tmp +install /tmp/azcopy /usr/local/bin/azcopy + +# Create azcopy 10 alias for backward compatibility +ln -sf /usr/local/bin/azcopy /usr/local/bin/azcopy10 diff --git a/images/ubuntu-slim/scripts/build/install-azure-cli.sh b/images/ubuntu-slim/scripts/build/install-azure-cli.sh new file mode 100644 index 000000000..df591179e --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-azure-cli.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e +################################################################################ +## File: install-azure-cli.sh +## Desc: Install Azure CLI (az) +################################################################################ + +# Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +curl -fsSL https://aka.ms/InstallAzureCLIDeb | sudo bash + +echo "azure-cli https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-linux?pivots=apt" >> $HELPER_SCRIPTS/apt-sources.txt + +rm -f /etc/apt/sources.list.d/azure-cli.list +rm -f /etc/apt/sources.list.d/azure-cli.list.save diff --git a/images/ubuntu-slim/scripts/build/install-azure-devops-cli.sh b/images/ubuntu-slim/scripts/build/install-azure-devops-cli.sh new file mode 100644 index 000000000..34ceee39a --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-azure-devops-cli.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +################################################################################ +## File: install-azure-devops-cli.sh +## Desc: Install Azure DevOps CLI (az devops) +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/etc-environment.sh + +# AZURE_EXTENSION_DIR shell variable defines where modules are installed +# https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview +export AZURE_EXTENSION_DIR=/opt/az/azcliextensions +set_etc_environment_variable "AZURE_EXTENSION_DIR" "${AZURE_EXTENSION_DIR}" + +# install azure devops Cli extension +az extension add -n azure-devops diff --git a/images/ubuntu-slim/scripts/build/install-bicep.sh b/images/ubuntu-slim/scripts/build/install-bicep.sh new file mode 100644 index 000000000..c23cfb280 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-bicep.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +################################################################################ +## File: install-bicep.sh +## Desc: Install bicep cli +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Install Bicep CLI +download_url=$(resolve_github_release_asset_url "Azure/bicep" "endswith(\"bicep-linux-x64\")" "latest") +bicep_binary_path=$(download_with_retry "${download_url}") + +# Mark it as executable +install "$bicep_binary_path" /usr/local/bin/bicep diff --git a/images/ubuntu-slim/scripts/build/install-git-lfs.sh b/images/ubuntu-slim/scripts/build/install-git-lfs.sh new file mode 100644 index 000000000..a8614a817 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-git-lfs.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +################################################################################ +## File: install-git-lfs.sh +## Desc: Install Git-lfs +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +GIT_LFS_REPO="https://packagecloud.io/install/repositories/github/git-lfs" + +# Install git-lfs +curl -fsSL $GIT_LFS_REPO/script.deb.sh | bash +apt-get install git-lfs + +# Remove source repo's +rm /etc/apt/sources.list.d/github_git-lfs.list + +# Document apt source repo's +echo "git-lfs $GIT_LFS_REPO" >> $HELPER_SCRIPTS/apt-sources.txt diff --git a/images/ubuntu-slim/scripts/build/install-git.sh b/images/ubuntu-slim/scripts/build/install-git.sh new file mode 100644 index 000000000..2bbd23e26 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-git.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e +################################################################################ +## File: install-git.sh +## Desc: Install Git and Git-FTP +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +GIT_REPO="ppa:git-core/ppa" + +## Install git +add-apt-repository $GIT_REPO -y +apt-get update +apt-get install git + +# Git version 2.35.2 introduces security fix that breaks action\checkout https://github.com/actions/checkout/issues/760 +cat <> /etc/gitconfig +[safe] + directory = * +EOF + +# Install git-ftp +apt-get install git-ftp + +# Remove source repo's +add-apt-repository --remove $GIT_REPO + +# Document apt source repo's +echo "git-core $GIT_REPO" >> $HELPER_SCRIPTS/apt-sources.txt + +# Add well-known SSH host keys to known_hosts +ssh-keyscan -t rsa,ecdsa,ed25519 github.com >> /etc/ssh/ssh_known_hosts +ssh-keyscan -t rsa ssh.dev.azure.com >> /etc/ssh/ssh_known_hosts diff --git a/images/ubuntu-slim/scripts/build/install-github-cli.sh b/images/ubuntu-slim/scripts/build/install-github-cli.sh new file mode 100644 index 000000000..7d120c95a --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-github-cli.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +################################################################################ +## File: install-github-cli.sh +## Desc: Install GitHub CLI +## Must be run as non-root user after homebrew +## Supply chain security: GitHub CLI - checksum validation +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Download GitHub CLI +gh_cli_url=$(resolve_github_release_asset_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and endswith(\".deb\")" "latest") +gh_cli_deb_path=$(download_with_retry "$gh_cli_url") + +# Supply chain security - GitHub CLI +hash_url=$(resolve_github_release_asset_url "cli/cli" "endswith(\"checksums.txt\")" "latest") +external_hash=$(get_checksum_from_url "$hash_url" "linux_amd64.deb" "SHA256") +use_checksum_comparison "$gh_cli_deb_path" "$external_hash" + +# Install GitHub CLI +apt-get install "$gh_cli_deb_path" diff --git a/images/ubuntu-slim/scripts/build/install-google-cloud-cli.sh b/images/ubuntu-slim/scripts/build/install-google-cloud-cli.sh new file mode 100644 index 000000000..3c5b78626 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-google-cloud-cli.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +################################################################################ +## File: install-google-cloud-cli.sh +## Desc: Install the Google Cloud CLI +################################################################################ + +REPO_URL="https://packages.cloud.google.com/apt" + +# Install the Google Cloud CLI +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] $REPO_URL cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list +wget -qO- https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor > /usr/share/keyrings/cloud.google.gpg +apt-get update +apt-get install google-cloud-cli + +# remove apt +rm /etc/apt/sources.list.d/google-cloud-sdk.list +rm /usr/share/keyrings/cloud.google.gpg + +# add repo to the apt-sources.txt +echo "google-cloud-sdk $REPO_URL" >> $HELPER_SCRIPTS/apt-sources.txt diff --git a/images/ubuntu-slim/scripts/build/install-ms-repos.sh b/images/ubuntu-slim/scripts/build/install-ms-repos.sh new file mode 100644 index 000000000..b4a56e8bd --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-ms-repos.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +################################################################################ +## File: install-ms-repos.sh +## Desc: Install official Microsoft package repos for the distribution +################################################################################ + +os_label=$(lsb_release -rs) + +# Install Microsoft repository +wget https://packages.microsoft.com/config/ubuntu/$os_label/packages-microsoft-prod.deb +dpkg -i packages-microsoft-prod.deb + +# update +apt-get install apt-transport-https ca-certificates curl software-properties-common +apt-get update +apt-get dist-upgrade diff --git a/images/ubuntu-slim/scripts/build/install-nodejs.sh b/images/ubuntu-slim/scripts/build/install-nodejs.sh new file mode 100644 index 000000000..bb99ed6f7 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-nodejs.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e +################################################################################ +## File: install-nodejs.sh +## Desc: Install Node.js LTS and related tooling (Gulp, Grunt) +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Install default Node.js +default_version=$(get_toolset_value '.node.default') + +curl -fsSL https://raw.githubusercontent.com/tj/n/master/bin/n -o ~/n +sudo bash ~/n $default_version + +# Install node modules +node_modules=$(get_toolset_value '.node_modules[].name') +if [ -n "$node_modules" ]; then + npm install -g $node_modules +else + echo "No node modules to install" +fi + +# fix global modules installation as regular user +# related issue https://github.com/actions/runner-images/issues/3727 +sudo chmod -R 777 /usr/local/lib/node_modules +sudo chmod -R 777 /usr/local/bin + +rm -rf ~/n diff --git a/images/ubuntu-slim/scripts/build/install-nvm.sh b/images/ubuntu-slim/scripts/build/install-nvm.sh new file mode 100644 index 000000000..c43273e2b --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-nvm.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +################################################################################ +## File: install-nvm.sh +## Desc: Install Nvm +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/etc-environment.sh + +export NVM_DIR="/etc/skel/.nvm" +mkdir ${NVM_DIR} +nvm_version=$(curl -fsSL https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') +curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/$nvm_version/install.sh | bash +set_etc_environment_variable "NVM_DIR" '$HOME/.nvm' + +echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' | tee -a /etc/skel/.bash_profile +[ -s "${NVM_DIR}/nvm.sh" ] && \. "${NVM_DIR}/nvm.sh" + +echo "source ${NVM_DIR}/nvm.sh" | tee -a /etc/skel/.bashrc + +# set system node.js as default one +nvm alias default system \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/build/install-pipx-packages.sh b/images/ubuntu-slim/scripts/build/install-pipx-packages.sh new file mode 100644 index 000000000..9dc496f01 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-pipx-packages.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e +################################################################################ +## File: install-pipx-packages.sh +## Desc: Install tools via pipx +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +export PATH="$PATH:/opt/pipx_bin" + +pipx_packages=$(get_toolset_value ".pipx[] .package") + +if [ -z "$pipx_packages" ]; then + echo "No pipx packages defined in toolset. Skipping pipx installation." + exit 0 +fi + +for package in $pipx_packages; do + echo "Install $package into default python" + pipx install $package + + # https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html + # Install ansible into an existing ansible-core Virtual Environment + if [[ $package == "ansible-core" ]]; then + pipx inject $package ansible + fi +done diff --git a/images/ubuntu-slim/scripts/build/install-powershell.sh b/images/ubuntu-slim/scripts/build/install-powershell.sh new file mode 100644 index 000000000..f4e4e29ea --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-powershell.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +################################################################################ +## File: install-powershell.sh +## Desc: Install PowerShell Core +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh +source $HELPER_SCRIPTS/os.sh + +pwsh_version=$(get_toolset_value .pwsh.version) + +# Install Powershell + + apt-get install powershell=$pwsh_version* \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/build/install-python.sh b/images/ubuntu-slim/scripts/build/install-python.sh new file mode 100644 index 000000000..e208b3a3e --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-python.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e +################################################################################ +## File: install-python.sh +## Desc: Install Python 3 +################################################################################ + +set -e +# Source the helpers for use with the script +source $HELPER_SCRIPTS/etc-environment.sh +source $HELPER_SCRIPTS/os.sh + +# Install Python, Python 3, pip, pip3 +apt-get install -y --no-install-recommends python3 python3-dev python3-pip python3-venv + +if is_ubuntu24; then +# Create temporary workaround to allow user to continue using pip + sudo cat < /etc/pip.conf +[global] +break-system-packages = true +EOF +fi + +# Install pipx +# Set pipx custom directory +export PIPX_BIN_DIR=/opt/pipx_bin +export PIPX_HOME=/opt/pipx + +python3 -m pip install pipx +python3 -m pipx ensurepath + +# Update /etc/environment +set_etc_environment_variable "PIPX_BIN_DIR" $PIPX_BIN_DIR +set_etc_environment_variable "PIPX_HOME" $PIPX_HOME +prepend_etc_environment_path $PIPX_BIN_DIR + +# Adding this dir to PATH will make installed pip commands are immediately available. +prepend_etc_environment_path '$HOME/.local/bin' \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/build/install-yq.sh b/images/ubuntu-slim/scripts/build/install-yq.sh new file mode 100644 index 000000000..40f978c88 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-yq.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +################################################################################ +## File: install-yq.sh +## Desc: Install yq - a command-line YAML, JSON and XML processor +## Supply chain security: yq - checksum validation +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Download yq +yq_url=$(resolve_github_release_asset_url "mikefarah/yq" "endswith(\"yq_linux_amd64\")" "latest") +binary_path=$(download_with_retry "${yq_url}") + +# Supply chain security - yq +hash_url=$(resolve_github_release_asset_url "mikefarah/yq" "endswith(\"checksums\")" "latest") +external_hash=$(get_checksum_from_url "${hash_url}" "yq_linux_amd64 " "SHA256" "true" " " "19") +use_checksum_comparison "$binary_path" "$external_hash" + +# Install yq +install "$binary_path" /usr/bin/yq + diff --git a/images/ubuntu-slim/scripts/build/install-zstd.sh b/images/ubuntu-slim/scripts/build/install-zstd.sh new file mode 100644 index 000000000..fcb5768c9 --- /dev/null +++ b/images/ubuntu-slim/scripts/build/install-zstd.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e +################################################################################ +## File: install-zstd.sh +## Desc: Install zstd +## Supply chain security: zstd - checksum validation +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/install.sh + +# Download zstd +release_tag=$(curl -fsSL https://api.github.com/repos/facebook/zstd/releases/latest | jq -r '.tag_name') +release_name="zstd-${release_tag//v}" +download_url="https://github.com/facebook/zstd/releases/download/${release_tag}/${release_name}.tar.gz" +archive_path=$(download_with_retry "${download_url}") + +# Supply chain security - zstd +external_hash=$(get_checksum_from_url "${download_url}.sha256" "${release_name}.tar.gz" "SHA256") +use_checksum_comparison "$archive_path" "$external_hash" + +# Install zstd +apt-get install liblz4-dev +tar xzf "$archive_path" -C /tmp + +make -C "/tmp/${release_name}/contrib/pzstd" -j $(nproc) all +make -C "/tmp/${release_name}" -j $(nproc) zstd-release + +for copyprocess in zstd zstdless zstdgrep; do + cp "/tmp/${release_name}/programs/${copyprocess}" /usr/local/bin/ +done + +cp "/tmp/${release_name}/contrib/pzstd/pzstd" /usr/local/bin/ + +for symlink in zstdcat zstdmt unzstd; do + ln -sf /usr/local/bin/zstd /usr/local/bin/${symlink} +done diff --git a/images/ubuntu-slim/scripts/docs-gen/Common.Helpers.psm1 b/images/ubuntu-slim/scripts/docs-gen/Common.Helpers.psm1 new file mode 100644 index 000000000..5f64d2b9e --- /dev/null +++ b/images/ubuntu-slim/scripts/docs-gen/Common.Helpers.psm1 @@ -0,0 +1,152 @@ +function Get-CommandResult { + <# + .SYNOPSIS + Runs a command in bash and returns the output and exit code. + + .DESCRIPTION + Function runs a provided command in bash and returns the output and exit code as hashtable. + + .PARAMETER Command + The command to run. + + .PARAMETER ExpectedExitCode + The expected exit code. If the actual exit code does not match, an exception is thrown. + + .PARAMETER Multiline + If true, the output is returned as an array of strings. Otherwise, the output is returned as a single string. + + .PARAMETER ValidateExitCode + If true, the actual exit code is compared to the expected exit code. + + .EXAMPLE + $result = Get-CommandResult "ls -la" + + This command runs "ls -la" in bash and returns the output and exit code as hashtable. + + #> + param( + [Parameter(Mandatory=$true)] + [string] $Command, + [int[]] $ExpectedExitCode = 0, + [switch] $Multiline, + [bool] $ValidateExitCode = $true + ) + + # Bash trick to suppress and show error output because some commands write to stderr (for example, "python --version") + $stdout = & bash -c "$Command 2>&1" + $exitCode = $LASTEXITCODE + + if ($ValidateExitCode) { + if ($ExpectedExitCode -notcontains $exitCode) { + try { + throw "StdOut: '$stdout' ExitCode: '$exitCode'" + } catch { + Write-Host $_.Exception.Message + Write-Host $_.ScriptStackTrace + exit $LASTEXITCODE + } + } + } + + return @{ + Output = If ($Multiline -eq $true) { $stdout } else { [string] $stdout } + ExitCode = $exitCode + } +} + +function Test-IsUbuntu22 { + return (lsb_release -rs) -eq "22.04" +} + +function Test-IsUbuntu24 { + return (lsb_release -rs) -eq "24.04" +} + +function Get-ToolsetContent { + <# + .SYNOPSIS + Retrieves the content of the toolset.json file. + + .DESCRIPTION + This function reads the toolset.json in path provided by INSTALLER_SCRIPT_FOLDER + environment variable and returns the content as a PowerShell object. + #> + + $toolsetPath = Join-Path $env:INSTALLER_SCRIPT_FOLDER "toolset.json" + $toolsetJson = Get-Content -Path $toolsetPath -Raw + ConvertFrom-Json -InputObject $toolsetJson +} + +function Invoke-DownloadWithRetry { + <# + .SYNOPSIS + Downloads a file from a given URL with retry functionality. + + .DESCRIPTION + The Invoke-DownloadWithRetry function downloads a file from the specified URL + to the specified path. It includes retry functionality in case the download fails. + + .PARAMETER Url + The URL of the file to download. + + .PARAMETER Path + The path where the downloaded file will be saved. If not provided, a temporary path + will be used. + + .EXAMPLE + Invoke-DownloadWithRetry -Url "https://example.com/file.zip" -Path "/usr/local/bin" + Downloads the file from the specified URL and saves it to the specified path. + + .EXAMPLE + Invoke-DownloadWithRetry -Url "https://example.com/file.zip" + Downloads the file from the specified URL and saves it to a temporary path. + + .OUTPUTS + The path where the downloaded file is saved. + #> + param( + [Parameter(Mandatory)] + [string] $Url, + [Alias("Destination")] + [string] $DestinationPath + ) + + if (-not $DestinationPath) { + $invalidChars = [IO.Path]::GetInvalidFileNameChars() -join '' + $re = "[{0}]" -f [RegEx]::Escape($invalidChars) + $fileName = [IO.Path]::GetFileName($Url) -replace $re + + if ([String]::IsNullOrEmpty($fileName)) { + $fileName = [System.IO.Path]::GetRandomFileName() + } + $DestinationPath = Join-Path -Path "/tmp" -ChildPath $fileName + } + + Write-Host "Downloading package from $Url to $DestinationPath..." + + $interval = 30 + $downloadStartTime = Get-Date + for ($retries = 20; $retries -gt 0; $retries--) { + try { + $attemptStartTime = Get-Date + Invoke-WebRequest -Uri $Url -Outfile $DestinationPath + $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2) + Write-Host "Package downloaded in $attemptSeconds seconds" + break + } catch { + $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2) + Write-Warning "Package download failed in $attemptSeconds seconds" + Write-Warning $_.Exception.Message + } + + if ($retries -eq 0) { + $totalSeconds = [math]::Round(($(Get-Date) - $downloadStartTime).TotalSeconds, 2) + throw "Package download failed after $totalSeconds seconds" + } + + Write-Warning "Waiting $interval seconds before retrying (retries left: $retries)..." + Start-Sleep -Seconds $interval + } + + return $DestinationPath +} diff --git a/images/ubuntu-slim/scripts/docs-gen/Generate-SoftwareReport.ps1 b/images/ubuntu-slim/scripts/docs-gen/Generate-SoftwareReport.ps1 new file mode 100644 index 000000000..021a0920b --- /dev/null +++ b/images/ubuntu-slim/scripts/docs-gen/Generate-SoftwareReport.ps1 @@ -0,0 +1,75 @@ +using module ../software-report-base/SoftwareReport.psm1 +using module ../software-report-base/SoftwareReport.Nodes.psm1 + +param ( + [Parameter(Mandatory)] + [string] $OutputDirectory +) + +$global:ErrorActionPreference = "Stop" +$global:ErrorView = "NormalView" +Set-StrictMode -Version Latest + +Import-Module (Join-Path $PSScriptRoot "SoftwareReport.Common.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "SoftwareReport.Helpers.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "Common.Helpers.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "SoftwareReport.Tools.psm1") -DisableNameChecking + +# Restore file owner in user profile +sudo chown -R ${env:USER}: $env:HOME + +# Software report +$softwareReport = [SoftwareReport]::new("Ubuntu-Slim") +$softwareReport.Root.AddToolVersion("OS Version:", $(Get-OSVersionFull)) +$softwareReport.Root.AddToolVersion("Kernel Version:", $(Get-KernelVersion)) +$softwareReport.Root.AddToolVersion("Image Version:", $env:IMAGE_VERSION) +$softwareReport.Root.AddToolVersion("Systemd version:", $(Get-SystemdVersion)) + +$installedSoftware = $softwareReport.Root.AddHeader("Installed Software") + +# Language and Runtime +$languageAndRuntime = $installedSoftware.AddHeader("Language and Runtime") +$languageAndRuntime.AddToolVersion("Bash", $(Get-BashVersion)) +$languageAndRuntime.AddToolVersion("Dash", $(Get-DashVersion)) +$languageAndRuntime.AddToolVersion("Node.js", $(Get-NodeVersion)) +$languageAndRuntime.AddToolVersion("Perl", $(Get-PerlVersion)) +$languageAndRuntime.AddToolVersion("Python", $(Get-PythonVersion)) + +# Package Management +$packageManagement = $installedSoftware.AddHeader("Package Management") +$packageManagement.AddToolVersion("Npm", $(Get-NpmVersion)) +$packageManagement.AddToolVersion("Pip", $(Get-PipVersion)) +$packageManagement.AddToolVersion("Pip3", $(Get-Pip3Version)) +$packageManagement.AddToolVersion("Pipx", $(Get-PipxVersion)) + +# Tools +$tools = $installedSoftware.AddHeader("Tools") +$tools.AddToolVersion("AzCopy", $(Get-AzCopyVersion)) +$tools.AddToolVersion("Bicep", $(Get-BicepVersion)) +$tools.AddToolVersion("Git", $(Get-GitVersion)) +$tools.AddToolVersion("Git LFS", $(Get-GitLFSVersion)) +$tools.AddToolVersion("Git-ftp", $(Get-GitFTPVersion)) +$tools.AddToolVersion("jq", $(Get-JqVersion)) +$tools.AddToolVersion("nvm", $(Get-NvmVersion)) +$tools.AddToolVersion("OpenSSL", $(Get-OpensslVersion)) +$tools.AddToolVersion("yq", $(Get-YqVersion)) +$tools.AddToolVersion("zstd", $(Get-ZstdVersion)) + +# CLI Tools +$cliTools = $installedSoftware.AddHeader("CLI Tools") +$cliTools.AddToolVersion("AWS CLI", $(Get-AWSCliVersion)) +$cliTools.AddToolVersion("AWS CLI Session Manager Plugin", $(Get-AWSCliSessionManagerPluginVersion)) +$cliTools.AddToolVersion("AWS SAM CLI", $(Get-AWSSAMVersion)) +$cliTools.AddToolVersion("Azure CLI", $(Get-AzureCliVersion)) +$cliTools.AddToolVersion("Azure CLI (azure-devops)", $(Get-AzureDevopsVersion)) +$cliTools.AddToolVersion("GitHub CLI", $(Get-GitHubCliVersion)) +$cliTools.AddToolVersion("Google Cloud CLI", $(Get-GoogleCloudCLIVersion)) + +# PowerShell Tools +$powerShellTools = $installedSoftware.AddHeader("PowerShell Tools") +$powerShellTools.AddToolVersion("PowerShell", $(Get-PowershellVersion)) + +$installedSoftware.AddHeader("Installed apt packages").AddTable($(Get-AptPackages)) + +$softwareReport.ToJson() | Out-File -FilePath "${OutputDirectory}/software-report.json" -Encoding UTF8NoBOM +$softwareReport.ToMarkdown() | Out-File -FilePath "${OutputDirectory}/software-report.md" -Encoding UTF8NoBOM diff --git a/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Common.psm1 b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Common.psm1 new file mode 100644 index 000000000..2c66bb72a --- /dev/null +++ b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Common.psm1 @@ -0,0 +1,81 @@ +function Get-BashVersion { + $version = bash -c 'echo ${BASH_VERSION}' + return $version +} + +function Get-DashVersion { + $version = dpkg-query -W -f '${Version}' dash + return $version +} + +function Get-NodeVersion { + $nodeVersion = $(node --version).Substring(1) + return $nodeVersion +} + +function Get-OpensslVersion { + $opensslVersion = $(dpkg-query -W -f '${Version}' openssl) + return $opensslVersion +} + +function Get-PerlVersion { + $version = $(perl -e 'print substr($^V,1)') + return $version +} + +function Get-PythonVersion { + $result = Get-CommandResult "python --version" + $version = $result.Output | Get-StringPart -Part 1 + return $version +} + +function Get-PowershellVersion { + $pwshVersion = $(pwsh --version) | Get-StringPart -Part 1 + return $pwshVersion +} + +function Get-NpmVersion { + $npmVersion = npm --version + return $npmVersion +} + +function Get-PipVersion { + $pipVersion = pip --version | Get-StringPart -Part 1 + return $pipVersion +} + +function Get-Pip3Version { + $pip3Version = pip3 --version | Get-StringPart -Part 1 + return $pip3Version +} + +function Get-AptPackages { + $apt = (Get-ToolsetContent).Apt + $output = @() + ForEach ($pkg in ($apt.vital_packages + $apt.common_packages + $apt.cmd_packages)) { + $version = $(dpkg-query -W -f '${Version}' $pkg) + if ($null -eq $version) { + $version = $(dpkg-query -W -f '${Version}' "$pkg*") + } + + $version = $version -replace '~','\~' + + $output += [PSCustomObject] @{ + Name = $pkg + Version = $version + } + } + return ($output | Sort-Object Name) +} + +function Get-PipxVersion { + $result = (Get-CommandResult "pipx --version").Output + $result -match "(?\d+\.\d+\.\d+\.?\d*)" | Out-Null + return $Matches.Version +} + +function Get-SystemdVersion { + $matchCollection = [regex]::Matches((systemctl --version | head -n 1), "\((.*?)\)") + $result = foreach ($match in $matchCollection) {$match.Groups[1].Value} + return $result +} diff --git a/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Helpers.psm1 b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Helpers.psm1 new file mode 100644 index 000000000..c74b85610 --- /dev/null +++ b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Helpers.psm1 @@ -0,0 +1,37 @@ +function Get-StringPart { + param ( + [Parameter(ValueFromPipeline)] + [string] $ToolOutput, + [string] $Delimiter = " ", + [int[]] $Part + ) + + $parts = $ToolOutput.Split($Delimiter, [System.StringSplitOptions]::RemoveEmptyEntries) + $selectedParts = $parts[$Part] + return [string]::Join($Delimiter, $selectedParts) +} + +function Get-PathWithLink { + param ( + [string] $InputPath + ) + + $link = Get-Item $InputPath | Select-Object -ExpandProperty Target + if (-not [string]::IsNullOrEmpty($link)) { + return "${InputPath} -> ${link}" + } + return "${InputPath}" +} + +function Get-OSVersionShort { + $(Get-OSVersionFull) | Get-StringPart -Delimiter '.' -Part 0,1 +} + +function Get-OSVersionFull { + lsb_release -ds | Get-StringPart -Part 1, 2 +} + +function Get-KernelVersion { + $kernelVersion = uname -r + return $kernelVersion +} diff --git a/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Tools.psm1 b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Tools.psm1 new file mode 100644 index 000000000..bca964913 --- /dev/null +++ b/images/ubuntu-slim/scripts/docs-gen/SoftwareReport.Tools.psm1 @@ -0,0 +1,79 @@ +function Get-AzCopyVersion { + $azcopyVersion = [string]$(azcopy --version) | Get-StringPart -Part 2 + return "$azcopyVersion - available by ``azcopy`` and ``azcopy10`` aliases" +} + +function Get-BicepVersion { + (bicep --version | Out-String) -match "bicep cli version (?\d+\.\d+\.\d+)" | Out-Null + return $Matches.Version +} + +function Get-GitVersion { + $gitVersion = git --version | Get-StringPart -Part -1 + return $gitVersion +} + +function Get-GitLFSVersion { + $result = Get-CommandResult "git-lfs --version" + $gitlfsversion = $result.Output | Get-StringPart -Part 0 | Get-StringPart -Part 1 -Delimiter "/" + return $gitlfsversion +} + +function Get-GitFTPVersion { + $gitftpVersion = git-ftp --version | Get-StringPart -Part 2 + return $gitftpVersion +} + +function Get-GoogleCloudCLIVersion { + return (gcloud --version | Select-Object -First 1) | Get-StringPart -Part 3 +} + +function Get-NvmVersion { + $nvmVersion = bash -c "source /etc/skel/.nvm/nvm.sh && nvm --version" + return $nvmVersion +} + +function Get-JqVersion { + $jqVersion = jq --version | Get-StringPart -Part 1 -Delimiter "-" + return $jqVersion +} + +function Get-AzureCliVersion { + $azcliVersion = (az version | ConvertFrom-Json).'azure-cli' + return $azcliVersion +} + +function Get-AzureDevopsVersion { + $azdevopsVersion = (az version | ConvertFrom-Json).extensions.'azure-devops' + return $azdevopsVersion +} + +function Get-AWSCliVersion { + $result = Get-CommandResult "aws --version" + $awsVersion = $result.Output | Get-StringPart -Part 0 | Get-StringPart -Part 1 -Delimiter "/" + return $awsVersion +} + +function Get-AWSCliSessionManagerPluginVersion { + $result = (Get-CommandResult "session-manager-plugin --version").Output + return $result +} + +function Get-AWSSAMVersion { + return $(sam --version | Get-StringPart -Part -1) +} + +function Get-GitHubCliVersion { + $ghVersion = gh --version | Select-String "gh version" | Get-StringPart -Part 2 + return $ghVersion +} + +function Get-ZstdVersion { + $zstdVersion = zstd --version | Get-StringPart -Part 1 -Delimiter "v" | Get-StringPart -Part 0 -Delimiter "," + return "$zstdVersion" +} + +function Get-YqVersion { + $yqVersion = $(yq -V) | Get-StringPart -Part 3 + return $yqVersion.TrimStart("v").Trim() +} diff --git a/images/ubuntu-slim/scripts/entrypoint.sh b/images/ubuntu-slim/scripts/entrypoint.sh new file mode 100644 index 000000000..b5c8076a8 --- /dev/null +++ b/images/ubuntu-slim/scripts/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# /opt/entrypoint.sh + +# Load environment variables from file +set -a +source /etc/environment +set +a + +# Execute the actual command +exec "$@" \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/helpers/cleanup.sh b/images/ubuntu-slim/scripts/helpers/cleanup.sh new file mode 100644 index 000000000..2b3daefba --- /dev/null +++ b/images/ubuntu-slim/scripts/helpers/cleanup.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +# delete all .gz and rotated file +find /var/log -type f -regex ".*\.gz$" -delete +find /var/log -type f -regex ".*\.[0-9]$" -delete + +# wipe log files +find /var/log/ -type f -exec cp /dev/null {} \; + +rm -rf /tmp/downloads /tmp/installers + +apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/helpers/etc-environment.sh b/images/ubuntu-slim/scripts/helpers/etc-environment.sh new file mode 100644 index 000000000..3261ccacb --- /dev/null +++ b/images/ubuntu-slim/scripts/helpers/etc-environment.sh @@ -0,0 +1,89 @@ +#!/bin/bash -e +################################################################################ +## File: etc-environment.sh +## Desc: Helper functions for source and modify /etc/environment +################################################################################ + +# NB: sed expression use '%' as a delimiter in order to simplify handling +# values containing slashes (i.e. directory path) +# The values containing '%' will break the functions + +get_etc_environment_variable() { + local variable_name=$1 + + # remove `variable_name=` and possible quotes from the line + grep "^${variable_name}=" /etc/environment | sed -E "s%^${variable_name}=\"?([^\"]+)\"?.*$%\1%" +} + +add_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + echo "${variable_name}=${variable_value}" | sudo tee -a /etc/environment +} + +replace_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + # modify /etc/environment in place by replacing a string that begins with variable_name + sudo sed -i -e "s%^${variable_name}=.*$%${variable_name}=${variable_value}%" /etc/environment +} + +set_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + if grep "^${variable_name}=" /etc/environment > /dev/null; then + replace_etc_environment_variable $variable_name $variable_value + else + add_etc_environment_variable $variable_name $variable_value + fi +} + +prepend_etc_environment_variable() { + local variable_name=$1 + local element=$2 + + # TODO: handle the case if the variable does not exist + existing_value=$(get_etc_environment_variable "${variable_name}") + set_etc_environment_variable "${variable_name}" "${element}:${existing_value}" +} + +append_etc_environment_variable() { + local variable_name=$1 + local element=$2 + + # TODO: handle the case if the variable does not exist + existing_value=$(get_etc_environment_variable "${variable_name}") + set_etc_environment_variable "${variable_name}" "${existing_value}:${element}" +} + +prepend_etc_environment_path() { + local element=$1 + + prepend_etc_environment_variable PATH "${element}" +} + +append_etc_environment_path() { + local element=$1 + + append_etc_environment_variable PATH "${element}" +} + +# Process /etc/environment as if it were shell script with `export VAR=...` expressions +# The PATH variable is handled specially in order to do not override the existing PATH +# variable. The value of PATH variable read from /etc/environment is added to the end +# of value of the exiting PATH variable exactly as it would happen with real PAM app read +# /etc/environment +# +# TODO: there might be the others variables to be processed in the same way as "PATH" variable +# ie MANPATH, INFOPATH, LD_*, etc. In the current implementation the values from /etc/evironments +# replace the values of the current environment +reload_etc_environment() { + # add `export ` to every variable of /etc/environment except PATH and eval the result shell script + eval $(grep -v '^PATH=' /etc/environment | sed -e 's%^%export %') + # handle PATH specially + etc_path=$(get_etc_environment_variable PATH) + export PATH="$PATH:$etc_path" +} diff --git a/images/ubuntu-slim/scripts/helpers/install.sh b/images/ubuntu-slim/scripts/helpers/install.sh new file mode 100644 index 000000000..cc71189da --- /dev/null +++ b/images/ubuntu-slim/scripts/helpers/install.sh @@ -0,0 +1,243 @@ +#!/bin/bash -e +################################################################################ +## File: install.sh +## Desc: Helper functions for installing tools +################################################################################ + +download_with_retry() { + local url=$1 + local download_path=$2 + + if [ -z "$download_path" ]; then + mkdir -p /tmp/downloads + download_path="/tmp/downloads/$(basename "$url")" + fi + + echo "Downloading package from $url to $download_path..." >&2 + + interval=30 + download_start_time=$(date +%s) + + for ((retries=20; retries>0; retries--)); do + attempt_start_time=$(date +%s) + if http_code=$(curl -4sSLo "$download_path" "$url" -w '%{http_code}'); then + attempt_seconds=$(($(date +%s) - attempt_start_time)) + if [ "$http_code" -eq 200 ]; then + echo "Package downloaded in $attempt_seconds seconds" >&2 + break + else + echo "Received HTTP status code $http_code after $attempt_seconds seconds" >&2 + fi + else + attempt_seconds=$(($(date +%s) - attempt_start_time)) + echo "Package download failed in $attempt_seconds seconds" >&2 + fi + + if [ "$retries" -le 1 ]; then + total_seconds=$(($(date +%s) - download_start_time)) + echo "Package download failed after $total_seconds seconds" >&2 + exit 1 + fi + + echo "Waiting $interval seconds before retrying (retries left: $retries)..." >&2 + sleep $interval + done + + echo "$download_path" +} + +get_github_releases_by_version() { + local repo=$1 + local version=${2:-".+"} + local allow_pre_release=${3:-false} + local with_assets_only=${4:-false} + + page_size="100" + + json=$(curl -fsSL "https://api.github.com/repos/${repo}/releases?per_page=${page_size}") + + if [[ -z "$json" ]]; then + echo "Failed to get releases" >&2 + exit 1 + fi + + if [[ $with_assets_only == "true" ]]; then + json=$(echo $json | jq -r '.[] | select(.assets | length > 0)') + else + json=$(echo $json | jq -r '.[]') + fi + + if [[ $allow_pre_release == "true" ]]; then + json=$(echo $json | jq -r '.') + else + json=$(echo $json | jq -r '. | select(.prerelease==false)') + fi + + # Filter out rc/beta/etc releases, convert to numeric version and sort + json=$(echo $json | jq '. | select(.tag_name | test(".*-[a-z]|beta") | not)' | jq '.tag_name |= gsub("[^\\d.]"; "")' | jq -s 'sort_by(.tag_name | split(".") | map(tonumber))') + + # Select releases matching version + if [[ $version == "latest" ]]; then + json_filtered=$(echo $json | jq .[-1]) + elif [[ $version == *"+"* ]] || [[ $version == *"*"* ]]; then + json_filtered=$(echo $json | jq --arg version $version '.[] | select(.tag_name | test($version))') + else + json_filtered=$(echo $json | jq --arg version $version '.[] | select(.tag_name | contains($version))') + fi + + if [[ -z "$json_filtered" ]]; then + echo "Failed to get releases from ${repo} matching version ${version}" >&2 + echo "Available versions: $(echo "$json" | jq -r '.tag_name')" >&2 + exit 1 + fi + + echo $json_filtered +} + +resolve_github_release_asset_url() { + local repo=$1 + local url_filter=$2 + local version=${3:-".+"} + local allow_pre_release=${4:-false} + local allow_multiple_matches=${5:-false} + + matching_releases=$(get_github_releases_by_version "${repo}" "${version}" "${allow_pre_release}" "true") + matched_url=$(echo $matching_releases | jq -r ".assets[].browser_download_url | select(${url_filter})") + + if [[ -z "$matched_url" ]]; then + echo "Found no download urls matching pattern: ${url_filter}" >&2 + echo "Available download urls: $(echo "$matching_releases" | jq -r '.assets[].browser_download_url')" >&2 + exit 1 + fi + + if [[ "$(echo "$matched_url" | wc -l)" -gt 1 ]]; then + if [[ $allow_multiple_matches == "true" ]]; then + matched_url=$(echo "$matched_url" | tail -n 1) + else + echo "Multiple matches found for ${version} version and ${url_filter} URL filter. Please make filters more specific" >&2 + exit 1 + fi + fi + + echo $matched_url +} + +get_checksum_from_github_release() { + local repo=$1 + local file_name=$2 + local version=${3:-".+"} + local hash_type=$4 + local allow_pre_release=${5:-false} + + if [[ -z "$file_name" ]]; then + echo "File name is not specified." >&2 + exit 1 + fi + + if [[ "$hash_type" == "SHA256" ]]; then + hash_pattern="[A-Fa-f0-9]{64}" + elif [[ "$hash_type" == "SHA512" ]]; then + hash_pattern="[A-Fa-f0-9]{128}" + else + echo "Unknown hash type: ${hash_type}" >&2 + exit 1 + fi + + matching_releases=$(get_github_releases_by_version "${repo}" "${version}" "${allow_pre_release}" "true") + matched_line=$(printf "$(echo $matching_releases | jq '.body')\n" | grep "$file_name") + + if [[ -z "$matched_line" ]]; then + echo "File name ${file_name} not found in release body" >&2 + exit 1 + fi + + if [[ "$(echo "$matched_line" | wc -l)" -gt 1 ]]; then + echo "Multiple matches found for ${file_name} in release body: ${matched_line}" >&2 + exit 1 + fi + + hash=$(echo $matched_line | grep -oP "$hash_pattern") + + if [[ -z "$hash" ]]; then + echo "Found ${file_name} in body of release, but failed to get hash from it: ${matched_line}" >&2 + exit 1 + fi + + echo "$hash" +} + +get_checksum_from_url() { + local url=$1 + local file_name=$2 + local hash_type=$3 + local use_custom_search_pattern=${4:-false} + local delimiter=${5:-' '} + local word_number=${6:-1} + + if [[ "$hash_type" == "SHA256" ]]; then + hash_pattern="[A-Fa-f0-9]{64}" + elif [[ "$hash_type" == "SHA512" ]]; then + hash_pattern="[A-Fa-f0-9]{128}" + else + echo "Unknown hash type: ${hash_type}" >&2 + exit 1 + fi + + checksums_file_path=$(download_with_retry "$url") + checksums=$(cat "$checksums_file_path") + rm "$checksums_file_path" + + matched_line=$(printf "$checksums\n" | grep "$file_name") + + if [[ "$(echo "$matched_line" | wc -l)" -gt 1 ]]; then + echo "Found multiple lines matching file name ${file_name} in checksum file." >&2 + exit 1 + fi + + if [[ -z "$matched_line" ]]; then + echo "File name ${file_name} not found in checksum file." >&2 + exit 1 + fi + + if [[ $use_custom_search_pattern == "true" ]]; then + hash=$(echo "$matched_line" | sed 's/ */ /g' | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]') + else + hash=$(echo $matched_line | grep -oP "$hash_pattern") + fi + + if [[ -z "$hash" ]]; then + echo "Found ${file_name} in checksum file, but failed to get hash from it: ${matched_line}" >&2 + exit 1 + fi + + echo "$hash" +} + +use_checksum_comparison() { + local file_path=$1 + local checksum=$2 + local sha_type=${3:-"256"} + + echo "Performing checksum verification" + + if [[ ! -f "$file_path" ]]; then + echo "File not found: $file_path" + exit 1 + fi + + local_file_hash=$(shasum --algorithm "$sha_type" "$file_path" | awk '{print $1}') + + if [[ "$local_file_hash" != "$checksum" ]]; then + echo "Checksum verification failed. Expected hash: $checksum; Actual hash: $local_file_hash." + exit 1 + else + echo "Checksum verification passed" + fi +} + +get_toolset_value() { + local toolset_path="${INSTALLER_SCRIPT_FOLDER}/toolset.json" + local query=$1 + + echo "$(jq -r "$query" $toolset_path)" +} \ No newline at end of file diff --git a/images/ubuntu-slim/scripts/helpers/os.sh b/images/ubuntu-slim/scripts/helpers/os.sh new file mode 100644 index 000000000..8699a9d9e --- /dev/null +++ b/images/ubuntu-slim/scripts/helpers/os.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e +################################################################################ +## File: os.sh +## Desc: Helper functions for OS releases +################################################################################ + +is_ubuntu22() { + lsb_release -rs | grep -q '22.04' +} + +is_ubuntu24() { + lsb_release -rs | grep -q '24.04' +} diff --git a/images/ubuntu-slim/test.sh b/images/ubuntu-slim/test.sh new file mode 100755 index 000000000..784d69154 --- /dev/null +++ b/images/ubuntu-slim/test.sh @@ -0,0 +1,93 @@ +#!/bin/bash -e + +# This script builds and runs various tests on the ubuntu-slim Docker image +# to ensure it contains the expected software and configurations. +# The build and test workflows for docker images expect this script to be present. +# +# Usage: test.sh [IMAGE_NAME] +# If IMAGE_NAME is not provided, defaults to ubuntu-slim:test + +show_help() { + echo "Usage: $0 [IMAGE_NAME]" + echo "" + echo "Test a Docker image to ensure it contains the expected software and configurations." + echo "" + echo "Arguments:" + echo " IMAGE_NAME Docker image name to test (default: ubuntu-slim:test)" + echo "" + echo "Examples:" + echo " $0 # Test ubuntu-slim:test (builds image first)" + echo " $0 my-registry/ubuntu:latest # Test existing image" + echo " $0 ubuntu-slim:v1.2.3 # Test tagged image" + echo "" + echo "Options:" + echo " -h, --help Show this help message" +} + +# Handle help flags +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + show_help + exit 0 +fi + +# Set the image name from parameter or use default +IMAGE_NAME="${1:-ubuntu-slim:test}" + +echo "Testing image: $IMAGE_NAME" + +run_test() { + local desc="$1" + shift + if output=$(docker run --rm "$IMAGE_NAME" "$@" 2>&1); then + echo "PASS: $desc" + echo "$output" | sed 's/^/ /' + else + echo "FAIL: $desc" + echo "$output" | sed 's/^/ /' + exit 1 + fi +} + +# Build the image only if using the default name (for backward compatibility) +if [[ "$IMAGE_NAME" == "ubuntu-slim:test" ]]; then + echo "Building image: $IMAGE_NAME" + docker build --debug --progress plain -t "$IMAGE_NAME" . +else + # Check if the image exists + if ! docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then + echo "Error: Image '$IMAGE_NAME' does not exist. Please build it first or provide a valid image name." + echo "Run '$0 --help' for usage information." + exit 1 + fi +fi + +echo "Running tests on image: $IMAGE_NAME" + +docker history --no-trunc "$IMAGE_NAME" +docker inspect -f "{{ .Size }}" "$IMAGE_NAME" | numfmt --to=iec | sed 's/^/Image size: /' + +# Ensure key software is installed and runnable +run_test "GitHub CLI is installed" gh --version +run_test "Azure CLI is installed" az version +run_test "AWS CLI is installed" aws --version +run_test "Session Manager plugin is installed" session-manager-plugin --version +run_test "AWS SAM CLI is installed" sam --version +run_test "jq is installed" jq --version +run_test "git is installed" git --version +run_test "node is installed" node --version +run_test "npm is installed" npm --version +run_test "python3 is installed" python3 --version +run_test "python is aliased" python --version +run_test "pipx is installed" pipx --version +run_test "curl is installed" curl --version +run_test "wget is installed" wget --version +run_test "yq is installed" yq --version +run_test "parallel is installed" parallel --version +run_test "bc is installed" bc --version +run_test "zstd is installed" zstd --version +run_test "google cloud SDK is installed" gcloud --version +run_test "git lfs is installed" git lfs version +run_test "powershell is installed" pwsh --version + +# Quick check: ensure the imagedata JSON file was created during image build +run_test "imagedata JSON file exists" test -f /imagegeneration/imagedata.json diff --git a/images/ubuntu-slim/toolsets/toolset.json b/images/ubuntu-slim/toolsets/toolset.json new file mode 100644 index 000000000..0e3ed0dd1 --- /dev/null +++ b/images/ubuntu-slim/toolsets/toolset.json @@ -0,0 +1,109 @@ +{ + "toolcache": [ + { + "name": "node", + "url" : "https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json", + "platform" : "linux", + "arch": "x64", + "versions": [ + "22.*", + "24.*" + ] + }, + { + "name": "CodeQL", + "platform" : "linux", + "arch": "x64", + "versions": [ + "*" + ] + } + ], + "apt": { + "vital_packages": [ + "apt-utils", + "bzip2", + "ca-certificates", + "curl", + "g++", + "gcc", + "make", + "jq", + "tar", + "unzip", + "wget" + ], + "common_packages": [ + "autoconf", + "automake", + "bc", + "dbus", + "dnsutils", + "dpkg", + "dpkg-dev", + "fakeroot", + "fonts-noto-color-emoji", + "gnupg2", + "iproute2", + "iputils-ping", + "libyaml-dev", + "libtool", + "libssl-dev", + "libsqlite3-dev", + "locales", + "lzma", + "mercurial", + "openssh-client", + "p7zip-rar", + "pkg-config", + "python-is-python3", + "rpm", + "texinfo", + "tk", + "tree", + "tzdata", + "upx", + "xvfb", + "xz-utils", + "zsync" + ], + "cmd_packages": [ + "acl", + "binutils", + "libnss3-tools", + "coreutils", + "file", + "findutils", + "flex", + "ftp", + "haveged", + "lz4", + "netcat-openbsd", + "net-tools", + "p7zip-full", + "parallel", + "patchelf", + "pigz", + "pollinate", + "rsync", + "shellcheck", + "sqlite3", + "ssh", + "sshpass", + "sudo", + "systemd-coredump", + "telnet", + "time", + "zip" + ] + }, + "brew": [ + ], + "node": { + "default": "24" + }, + "node_modules": [ ], + "pwsh": { + "version": "7.5" + } +} diff --git a/images/ubuntu-slim/ubuntu-slim-Readme.md b/images/ubuntu-slim/ubuntu-slim-Readme.md new file mode 100644 index 000000000..6e11dbb70 --- /dev/null +++ b/images/ubuntu-slim/ubuntu-slim-Readme.md @@ -0,0 +1,118 @@ +# Ubuntu-Slim +- OS Version: 24.04.3 LTS +- Kernel Version: 6.14.0-36-generic +- Image Version: 1.0.0 +- Systemd version: 255.4-1ubuntu8.11 + +## Installed Software + +### Language and Runtime +- Bash 5.2.21(1)-release +- Dash 0.5.12-6ubuntu5 +- Node.js 24.12.0 +- Perl 5.38.2 +- Python 3.12.3 + +### Package Management +- Npm 11.6.2 +- Pip 24.0 +- Pip3 24.0 +- Pipx 1.8.0 + +### Tools +- AzCopy 10.31.0 - available by `azcopy` and `azcopy10` aliases +- Bicep 0.39.26 +- Git 2.52.0 +- Git LFS 3.7.1 +- Git-ftp 1.6.0 +- jq 1.7 +- nvm 0.40.3 +- OpenSSL 3.0.13-0ubuntu3.6 +- yq 4.49.2 +- zstd 1.5.7 + +### CLI Tools +- AWS CLI 2.32.14 +- AWS CLI Session Manager Plugin 1.2.764.0 +- AWS SAM CLI 1.150.1 +- Azure CLI 2.81.0 +- Azure CLI (azure-devops) 1.0.2 +- GitHub CLI 2.83.2 +- Google Cloud CLI 549.0.1 + +### PowerShell Tools +- PowerShell 7.5.4 + +### Installed apt packages +| Name | Version | +| ---------------------- | ---------------------------- | +| acl | 2.3.2-1build1.1 | +| apt-utils | 2.8.3 | +| autoconf | 2.71-3 | +| automake | 1:1.16.5-1.3ubuntu1 | +| bc | 1.07.1-3ubuntu4 | +| binutils | 2.42-4ubuntu2.8 | +| bzip2 | 1.0.8-5.1build0.1 | +| ca-certificates | 20240203 | +| coreutils | 9.4-3ubuntu6.1 | +| curl | 8.5.0-2ubuntu10.6 | +| dbus | 1.14.10-4ubuntu4.1 | +| dnsutils | 1:9.18.39-0ubuntu0.24.04.2 | +| dpkg | 1.22.6ubuntu6.5 | +| dpkg-dev | 1.22.6ubuntu6.5 | +| fakeroot | 1.33-1 | +| file | 1:5.45-3build1 | +| findutils | 4.9.0-5build1 | +| flex | 2.6.4-8.2build1 | +| fonts-noto-color-emoji | 2.047-0ubuntu0.24.04.1 | +| ftp | 20230507-2build3 | +| g++ | 4:13.2.0-7ubuntu1 | +| gcc | 4:13.2.0-7ubuntu1 | +| gnupg2 | 2.4.4-2ubuntu17.3 | +| haveged | 1.9.14-1ubuntu2 | +| iproute2 | 6.1.0-1ubuntu6.2 | +| iputils-ping | 3:20240117-1ubuntu0.1 | +| jq | 1.7.1-3ubuntu0.24.04.1 | +| libnss3-tools | 2:3.98-1build1 | +| libsqlite3-dev | 3.45.1-1ubuntu2.5 | +| libssl-dev | 3.0.13-0ubuntu3.6 | +| libtool | 2.4.7-7build1 | +| libyaml-dev | 0.2.5-1build1 | +| locales | 2.39-0ubuntu8.6 | +| lz4 | 1.9.4-1build1.1 | +| lzma | 9.22-2.2 | +| make | 4.3-4.1build2 | +| mercurial | 6.7.2-1ubuntu2.2 | +| net-tools | 2.10-0.1ubuntu4.4 | +| netcat-openbsd | 1.226-1ubuntu2 | +| openssh-client | 1:9.6p1-3ubuntu13.14 | +| p7zip-full | 16.02+transitional.1 | +| p7zip-rar | 16.02+transitional.1 | +| parallel | 20231122+ds-1 | +| patchelf | 0.18.0-1.1build1 | +| pigz | 2.8-1 | +| pkg-config | 1.8.1-2build1 | +| pollinate | 4.33-3.1ubuntu1.1 | +| python-is-python3 | 3.11.4-1 | +| rpm | 4.18.2+dfsg-2.1build2 | +| rsync | 3.2.7-1ubuntu1.2 | +| shellcheck | 0.9.0-1 | +| sqlite3 | 3.45.1-1ubuntu2.5 | +| ssh | 1:9.6p1-3ubuntu13.14 | +| sshpass | 1.09-1 | +| sudo | 1.9.15p5-3ubuntu5.24.04.1 | +| systemd-coredump | 255.4-1ubuntu8.11 | +| tar | 1.35+dfsg-3build1 | +| telnet | 0.17+2.5-3ubuntu4 | +| texinfo | 7.1-3build2 | +| time | 1.9-0.2build1 | +| tk | 8.6.14build1 | +| tree | 2.1.1-2ubuntu3.24.04.2 | +| tzdata | 2025b-0ubuntu0.24.04.1 | +| unzip | 6.0-28ubuntu4.1 | +| upx | 4.2.2-3 | +| wget | 1.21.4-1ubuntu4.1 | +| xvfb | 2:21.1.12-1ubuntu1.5 | +| xz-utils | 5.6.1+really5.4.5-1ubuntu0.2 | +| zip | 3.0-13ubuntu0.2 | +| zsync | 0.6.2-5build1 | diff --git a/images/ubuntu/scripts/helpers/etc-environment.sh b/images/ubuntu/scripts/helpers/etc-environment.sh index 7a235b784..3261ccacb 100644 --- a/images/ubuntu/scripts/helpers/etc-environment.sh +++ b/images/ubuntu/scripts/helpers/etc-environment.sh @@ -26,7 +26,7 @@ replace_etc_environment_variable() { local variable_name=$1 local variable_value=$2 - # modify /etc/environemnt in place by replacing a string that begins with variable_name + # modify /etc/environment in place by replacing a string that begins with variable_name sudo sed -i -e "s%^${variable_name}=.*$%${variable_name}=${variable_value}%" /etc/environment } @@ -81,7 +81,7 @@ append_etc_environment_path() { # ie MANPATH, INFOPATH, LD_*, etc. In the current implementation the values from /etc/evironments # replace the values of the current environment reload_etc_environment() { - # add `export ` to every variable of /etc/environemnt except PATH and eval the result shell script + # add `export ` to every variable of /etc/environment except PATH and eval the result shell script eval $(grep -v '^PATH=' /etc/environment | sed -e 's%^%export %') # handle PATH specially etc_path=$(get_etc_environment_variable PATH)