diff options
Diffstat (limited to 'vcpkg/scripts/azure-pipelines/linux')
| -rw-r--r-- | vcpkg/scripts/azure-pipelines/linux/Dockerfile | 10 | ||||
| -rw-r--r-- | vcpkg/scripts/azure-pipelines/linux/azure-pipelines.yml | 147 | ||||
| -rwxr-xr-x | vcpkg/scripts/azure-pipelines/linux/provision-image.sh | 178 |
3 files changed, 335 insertions, 0 deletions
diff --git a/vcpkg/scripts/azure-pipelines/linux/Dockerfile b/vcpkg/scripts/azure-pipelines/linux/Dockerfile new file mode 100644 index 0000000..c07392b --- /dev/null +++ b/vcpkg/scripts/azure-pipelines/linux/Dockerfile @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.4
+# DisableDockerDetector "Used to build the container deployed to Azure Container Registry"
+FROM ubuntu:noble-20251001
+ADD provision-image.sh /provision-image.sh
+RUN apt-get update && \
+ apt-get install --no-install-recommends -y curl gnupg ca-certificates
+# The BUILD_DATE argument forces cache invalidation so we get updated apt dependencies
+ARG BUILD_DATE
+RUN echo "Build date: ${BUILD_DATE}"
+RUN chmod +x /provision-image.sh && /provision-image.sh
diff --git a/vcpkg/scripts/azure-pipelines/linux/azure-pipelines.yml b/vcpkg/scripts/azure-pipelines/linux/azure-pipelines.yml new file mode 100644 index 0000000..0da2b03 --- /dev/null +++ b/vcpkg/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +parameters: + - name: vcpkgToolSha + displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' + type: string + default: 'use default' + - name: jobName + type: string + default: 'x64_linux' + - name: tripletPattern + displayName: 'Enable the triplets which contain this substring' + type: string + default: '' + +jobs: +- job: ${{ parameters.jobName }} + condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) + pool: + name: PrAzureLinux3Docker + timeoutInMinutes: 1440 # 1 day + variables: + - name: WORKING_ROOT + value: /mnt/vcpkg-ci + - name: VCPKG_DOWNLOADS + value: /mnt/vcpkg-ci/downloads + - name: LINUX_DOCKER_IMAGE + value: 'vcpkgandroidwus.azurecr.io/vcpkg-linux:2025-10-16' + steps: + # Note: /mnt is the Azure machines' temporary disk. + - bash: | + sudo mkdir /home/agent -m=777 + sudo chown `id -u` /home/agent + sudo mkdir ${{ variables.WORKING_ROOT }} -m=777 + sudo rm -rf ${{ variables.WORKING_ROOT }}/failure-logs + sudo mkdir ${{ variables.WORKING_ROOT }}/failure-logs -m=777 + sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777 + # Move the docker layers to the temp disk. + sudo mkdir -p /etc/docker + echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + exit 0 + displayName: 'Create working directories' + - bash: ./bootstrap-vcpkg.sh -skipDependencyChecks + displayName: 'Bootstrap vcpkg' + condition: eq('use default', '${{ parameters.vcpkgToolSha }}') + - task: AzureCLI@2 + displayName: 'Build vcpkg with CMake' + condition: ne('use default', '${{ parameters.vcpkgToolSha }}') + inputs: + azureSubscription: 'vcpkg-pr-fleet-wus' + scriptType: bash + scriptLocation: 'inlineScript' + inlineScript: | + # This is a second pull but the vcpkgToolSha setting is used rarely. + USER=$(id --user) + az acr login --name vcpkgandroidwus + docker pull ${{ variables.LINUX_DOCKER_IMAGE }} + docker run --init -i --rm \ + -a stderr \ + -a stdout \ + --user $USER \ + --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \ + --workdir /vcpkg \ + ${{ variables.LINUX_DOCKER_IMAGE }} \ + /vcpkg/scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }} + - task: AzureCLI@2 + displayName: '*** Test Modified Ports' + inputs: + azureSubscription: 'vcpkg-pr-fleet-wus' + scriptType: bash + scriptLocation: 'inlineScript' # Be very very careful that the exit code from the last pwsh is reported correctly + inlineScript: | + end=`date -u -d "2 days" '+%Y-%m-%dT%H:%MZ'` + assetSas=`az storage container generate-sas --name cache --account-name vcpkgassetcachewus --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv` + binarySas=`az storage container generate-sas --name cache --account-name vcpkgbinarycachewus --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv` + echo Minting SAS tokens valid through $end + # Persist the binary SAS as a secret pipeline variable for the owners-db step + echo "##vso[task.setvariable variable=BCACHE_SAS_TOKEN;issecret=true]$binarySas" + USER=$(id --user) + az acr login --name vcpkgandroidwus + docker pull ${{ variables.LINUX_DOCKER_IMAGE }} + docker run --init -i --rm \ + -a stderr \ + -a stdout \ + --user $USER \ + --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \ + --mount type=bind,source=$(WORKING_ROOT)/failure-logs,target=/vcpkg/failure-logs \ + --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \ + --env X_VCPKG_ASSET_SOURCES="x-azurl,https://vcpkgassetcachewus.blob.core.windows.net/cache,$assetSas,readwrite" \ + --workdir /vcpkg \ + ${{ variables.LINUX_DOCKER_IMAGE }} \ + pwsh \ + -File scripts/azure-pipelines/test-modified-ports.ps1 \ + -Triplet ${{ replace(parameters.jobName, '_', '-') }} \ + -BuildReason $(Build.Reason) \ + -BinarySourceStub "x-azcopy-sas,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \ + -WorkingRoot ${{ variables.WORKING_ROOT }} + - task: PublishPipelineArtifact@1 + displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}" + inputs: + targetPath: '$(WORKING_ROOT)/failure-logs' + artifact: "failure logs for ${{ replace(parameters.jobName, '_', '-') }}" + condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') + - task: PublishPipelineArtifact@1 + displayName: "Publish Artifact: azcopy logs for ${{ replace(parameters.jobName, '_', '-') }}" + inputs: + targetPath: '$(WORKING_ROOT)/azcopy-logs' + artifactName: "z azcopy logs for ${{ replace(parameters.jobName, '_', '-') }}" + condition: ne(variables['AZCOPY_LOGS_EMPTY'], 'True') + - task: UseNode@1 + displayName: 'Ensure Node.js is available' + condition: always() + inputs: + version: '22.x' + - bash: | + cd scripts/azure-pipelines/owners-db && npm ci || true + # Construct the blob base url using the secret SAS token set earlier + blob="https://vcpkgbinarycachewus.blob.core.windows.net/cache?${BCACHE_SAS_TOKEN}" + if [ "$(Build.Reason)" = "PullRequest" ]; then + echo "Running file_script_from_cache for PR" + npx --yes ts-node ./file_script_from_cache.ts --pr-hashes "$(Build.Repository.LocalPath)/pr-hashes.json" --blob-base-url "$blob" --target-branch "origin/master" --out-dir ../../list_files + else + echo "Running file_script for CI" + npx --yes ts-node ./file_script.ts --info-dir /mnt/vcpkg-ci/installed/vcpkg/info/ --out-dir ../../list_files + fi + displayName: 'Build a file list for all packages' + condition: always() + env: + BCACHE_SAS_TOKEN: $(BCACHE_SAS_TOKEN) + - task: PublishPipelineArtifact@1 + displayName: "Publish Artifact: file lists for ${{ replace(parameters.jobName, '_', '-') }}" + condition: always() + inputs: + targetPath: scripts/list_files + artifact: "file lists for ${{ replace(parameters.jobName, '_', '-') }}" + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + condition: ne(variables['XML_RESULTS_FILE'], '') + inputs: + testRunTitle: ${{ replace(parameters.jobName, '_', '-') }} + testResultsFormat: xUnit + testResultsFiles: $(XML_RESULTS_FILE) + platform: ${{ replace(parameters.jobName, '_', '-') }} + configuration: static diff --git a/vcpkg/scripts/azure-pipelines/linux/provision-image.sh b/vcpkg/scripts/azure-pipelines/linux/provision-image.sh new file mode 100755 index 0000000..62daced --- /dev/null +++ b/vcpkg/scripts/azure-pipelines/linux/provision-image.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +export DEBIAN_FRONTEND=noninteractive + +# Add apt repos + +# Detect Ubuntu VERSION_ID from /etc/os-release (e.g., "24.04") and format to "2404" +UBUNTU_VERSION_ID=$(. /etc/os-release && echo "$VERSION_ID") +NVIDIA_REPO_VERSION=$(echo "$UBUNTU_VERSION_ID" | sed 's/\.//') + +# Apt dependencies; needed for add-apt-repository and curl downloads to work +apt-get -y update +apt-get --no-install-recommends -y install ca-certificates curl apt-transport-https lsb-release gnupg software-properties-common + +## CUDA +curl -L -o /etc/apt/preferences.d/cuda-repository-pin-600 "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${NVIDIA_REPO_VERSION}/x86_64/cuda-ubuntu${NVIDIA_REPO_VERSION}.pin" +apt-key adv --fetch-keys "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${NVIDIA_REPO_VERSION}/x86_64/3bf863cc.pub" +add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${NVIDIA_REPO_VERSION}/x86_64/ /" + +## PowerShell +curl -L -o packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/${UBUNTU_VERSION_ID}/packages-microsoft-prod.deb +dpkg -i packages-microsoft-prod.deb +rm -f packages-microsoft-prod.deb +add-apt-repository universe + +## Azure CLI +mkdir -p /etc/apt/keyrings +curl -sLS https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | + tee /etc/apt/keyrings/microsoft.gpg > /dev/null +chmod go+r /etc/apt/keyrings/microsoft.gpg + +AZ_DIST=$(lsb_release -cs) +echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_DIST main" | + tee /etc/apt/sources.list.d/azure-cli.list + +apt-get -y update +apt-get -y upgrade + +# Add apt packages + +## vcpkg prerequisites +APT_PACKAGES="git curl zip unzip tar" + +## essentials +APT_PACKAGES="$APT_PACKAGES \ + autoconf autoconf-archive \ + autopoint \ + build-essential \ + cmake \ + gcc g++ gfortran \ + libnuma1 libnuma-dev \ + libtool libtool-bin libltdl-dev \ + libudev-dev \ +" + +## vcpkg_find_acquire_program +APT_PACKAGES="$APT_PACKAGES \ + bison libbison-dev \ + flex \ + gperf \ + nasm \ + ninja-build \ + pkg-config \ + python3 \ + ruby-full \ + swig \ + yasm \ +" + +## mesa and X essentials +APT_PACKAGES="$APT_PACKAGES \ + mesa-common-dev libgl1-mesa-dev libglu1-mesa-dev libgles2-mesa-dev \ + libx11-dev \ + libxaw7-dev \ + libxcursor-dev \ + libxi-dev \ + libxinerama-dev \ + libxkbcommon-x11-dev \ + libxrandr-dev \ + libxt-dev \ + libxxf86vm-dev \ + xutils-dev \ +" + +## required by qt5-base +APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \ + libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev" + +## required by qt5-base for qt5-x11extras +APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \ + libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev \ + libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \ + libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev" + +## required by xcb feature in qtbase +APT_PACKAGES="$APT_PACKAGES libxcb-cursor-dev" + +## required by libhdfs3 +APT_PACKAGES="$APT_PACKAGES libkrb5-dev" + +## required by kf5windowsystem +APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev" + +## required by kf5globalaccel +APT_PACKAGES="$APT_PACKAGES libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev" + +## required by mesa +APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako libxcb-dri3-dev libxcb-present-dev" + +## required by some packages to install additional python packages +APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv python3-jinja2" + +## required by qtwebengine +APT_PACKAGES="$APT_PACKAGES nodejs" + +## required by qtwayland +APT_PACKAGES="$APT_PACKAGES libwayland-dev" + +## required by all GN projects +APT_PACKAGES="$APT_PACKAGES python-is-python3" + +## required by libctl +APT_PACKAGES="$APT_PACKAGES guile-2.2-dev" + +## required by gtk +APT_PACKAGES="$APT_PACKAGES libxdamage-dev libselinux1-dev" + +## required by at-spi2-atk +APT_PACKAGES="$APT_PACKAGES libxtst-dev" + +## required by boringssl +APT_PACKAGES="$APT_PACKAGES golang-go" + +## required by libdecor and mesa +APT_PACKAGES="$APT_PACKAGES wayland-protocols" + +## required by robotraconteur +APT_PACKAGES="$APT_PACKAGES libbluetooth-dev" + +## required by libmysql +APT_PACKAGES="$APT_PACKAGES libtirpc-dev" + +## CUDA +# The intent is to install everything that does not require an actual GPU, driver, or GUI. +# Intentionally omitted: cuda-demo-suite-12-9 cuda-documentation-12-9 cuda-driver-* +# cuda-gdb-12-9 cuda-gdb-src-12-9 cuda-nsight-* cuda-nvdisasm +# cuda-nvprof cuda-nvprune cuda-profiler-api* cuda-sandbox-* +# cuda-visual-tools-12-9 nvidia-gds-12-9 cuda-nvvp-12-9 +# cuda-toolkit-12-9 cuda-tools-12-9 cuda-command-line-tools-12-9 +# cuda-runtime-12-9 +# All libraries for which there is a -dev suffix included here +# cudnn9-jit-cuda-12-9 : Depends: libcudnn9-jit-dev-cuda-12 (= 9.12.0.46-1) but it is not installable +APT_PACKAGES="$APT_PACKAGES cuda-cccl-12-9 cuda-compat-12-9 cuda-compiler-12-9 cuda-crt-12-9 \ + cuda-cudart-dev-12-9 cuda-cuobjdump-12-9 cuda-cupti-dev-12-9 cuda-cuxxfilt-12-9 \ + cuda-driver-dev-12-9 cuda-libraries-dev-12-9 cuda-minimal-build-12-9 cuda-nvcc-12-9 \ + cuda-nvml-dev-12-9 cuda-nvrtc-dev-12-9 cuda-nvtx-12-9 cuda-nvvm-12-9 cuda-opencl-dev-12-9 \ + cuda-sanitizer-12-9 cuda-toolkit-12-9-config-common cudnn9-cuda-12-9 gds-tools-12-9 \ + libcublas-12-9 libcudnn9-dev-cuda-12 libcufft-dev-12-9 libcurand-dev-12-9 libcusolver-dev-12-9 \ + libcusparse-dev-12-9 libnccl-dev libnpp-dev-12-9 libnvfatbin-dev-12-9 libnvjitlink-dev-12-9 \ + libnvjpeg-dev-12-9" + +## PowerShell + Azure +APT_PACKAGES="$APT_PACKAGES powershell azcopy azure-cli" + +## Additionally required/installed by Azure DevOps Scale Set Agents, skip on WSL +if [[ $(grep microsoft /proc/version) ]]; then +echo "Skipping install of ADO prerequisites on WSL." +else +APT_PACKAGES="$APT_PACKAGES libkrb5-3 zlib1g libicu70 debsums liblttng-ust1" +fi + +apt-get --no-install-recommends -y install $APT_PACKAGES + +az --version |