source: git/.github/workflows/ci-sage.yml @ 7d6820

spielwiese
Last change on this file since 7d6820 was 7d6820, checked in by Matthias Koeppe <mkoeppe@…>, 3 years ago
.github/workflows/ci-sage.yml: Test also on macOS Big Sur and with different Xcode versions
  • Property mode set to 100644
File size: 15.3 KB
RevLine 
[d2074d]1name: Run Sage CI for Linux/Cygwin/macOS
2
3## This GitHub Actions workflow provides:
4##
5##  - portability testing, by building and testing this project on many platforms
6##    (Linux variants and Cygwin), each with two configurations (installed packages),
7##
8##  - continuous integration, by building and testing other software
9##    that depends on this project.
10##
11## It runs on every pull request and push of a tag to the GitHub repository.
12##
13## The testing can be monitored in the "Actions" tab of the GitHub repository.
14##
15## After all jobs have finished (or are canceled) and a short delay,
16## tar files of all logs are made available as "build artifacts".
17##
18## This GitHub Actions workflow uses the portability testing framework
19## of SageMath (https://www.sagemath.org/).  For more information, see
20## https://doc.sagemath.org/html/en/developer/portability_testing.html
21
22## The workflow consists of two jobs:
23##
24##  - First, it builds a source distribution of the project
25##    and generates a script "update-pkgs.sh".  It uploads them
26##    as a build artifact named upstream.
27##
28##  - Second, it checks out a copy of the SageMath source tree.
29##    It downloads the upstream artifact and replaces the project's
30##    package in the SageMath distribution by the newly packaged one
31##    from the upstream artifact, by running the script "update-pkgs.sh".
32##    Then it builds a small portion of the Sage distribution.
33##
34## Many copies of the second step are run in parallel for each of the tested
35## systems/configurations.
36
37#on: [push, pull_request]
38
39on:
40  pull_request:
41    types: [opened, synchronize]
42  push:
43    tags:
44      - '*'
45
46env:
47  # Ubuntu packages to install so that the project's "make dist" can succeed
[ab6949]48  DIST_PREREQ: libcdd-dev
[d2074d]49  # Name of this project in the Sage distribution
[a72a30]50  SPKG:        singular
[d2074d]51  # Sage distribution packages to build
52  TARGETS_PRE: build/make/Makefile
[a72a30]53  TARGETS:     SAGE_CHECK=yes singular
54  TARGETS_OPTIONAL: SAGE_CHECK=warn pynac pysingular
[d2074d]55  # Standard setting: Test the current beta release of Sage:
56  SAGE_REPO:   sagemath/sage
57  SAGE_REF:    develop
[68f6c9]58  # Temporarily test on the branch from sage ticket for singular 4.2 upgrade
[d2074d]59  SAGE_TRAC_GIT: git://trac.sagemath.org/sage.git
[efb3fc]60  SAGE_TRAC_COMMAND: try
[a72a30]61  SAGE_TICKET: 25993
[d2074d]62  REMOVE_PATCHES: "*"
63
64jobs:
65
66  dist:
67    runs-on: ubuntu-latest
68    steps:
69      - name: Check out ${{ env.SPKG }}
70        uses: actions/checkout@v2
71        with:
72          path: build/pkgs/${{ env.SPKG }}/src
73      - name: Install prerequisites
74        run: |
75          sudo DEBIAN_FRONTEND=noninteractive apt-get update
76          sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ
77      - name: Run make dist, prepare upstream artifact
78        run: |
[ab6949]79          (cd build/pkgs/${{ env.SPKG }}/src && (./autogen.sh && ./configure && make dist) ) \
[d2074d]80          && mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \
81          && echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=optional" > upstream/update-pkgs.sh \
82          && if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \
83          && ls -l upstream/
84      - uses: actions/upload-artifact@v2
85        with:
86          path: upstream
87          name: upstream
88
89  cygwin:
90    env:
91      STAGE: i-a
92      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
93      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
94      MAKE: make -j8
95      SAGE_NUM_THREADS: 3
96      SAGE_CHECK: yes
97      SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
98      CYGWIN: winsymlinks:native
99      CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
100      SAGE_FAT_BINARY: yes
[68f6c9]101      SAGE_LOCAL: /opt/sage-singular-${{ github.sha }}
[d2074d]102
103    runs-on: windows-latest
104
105    needs: [dist]
106
107    strategy:
108      fail-fast: false
109      matrix:
110        pkgs: [minimal, standard]
111    steps:
112    - run: |
113        git config --global core.autocrlf false
114        git config --global core.symlinks true
115    - name: install cygwin with choco
116      shell: bash {0}
117      run: |
118        choco --version
119        choco install git python3 --source cygwin
120    - name: Check out SageMath
121      uses: actions/checkout@v2
122      with:
123        repository: ${{ env.SAGE_REPO }}
124        ref: ${{ env.SAGE_REF }}
125      if: env.SAGE_REPO != ''
126    - name: Check out git-trac-command
127      uses: actions/checkout@v2
128      with:
129        repository: sagemath/git-trac-command
130        path: git-trac-command
131      if: env.SAGE_TRAC_GIT != ''
132    - name: Check out SageMath from trac.sagemath.org
133      shell: bash {0}
134      # Random sleep and retry to limit the load on trac.sagemath.org
135      run: |
[efb3fc]136        if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac ${{ env.SAGE_TRAC_COMMAND }} $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
[d2074d]137      if: env.SAGE_TRAC_GIT != ''
138    - uses: actions/download-artifact@v2
139      with:
140        path: upstream
141        name: upstream
142    - name: install minimal prerequisites with choco
143      shell: bash {0}
144      run: |
145        choco --version
[68f6c9]146        PACKAGES="python38 python38-pip"
[d2074d]147        choco install $PACKAGES --source cygwin
148    - name: Update Sage packages from upstream artifact
149      run: |
150        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && ls -l upstream/ && export PATH="$(pwd)/build/bin:$PATH:/usr/local/bin:/usr/bin" && (cd upstream && bash -x update-pkgs.sh) && git diff'
[68f6c9]151    - name: tox
[d2074d]152      run: |
[68f6c9]153        C:\\tools\\cygwin\\bin\\bash -l -x -c 'python3.8 -m pip install tox'
154        C:\\tools\\cygwin\\bin\\bash -l -x -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && PREFIX="${{ env.SAGE_LOCAL }}" tox -e local-cygwin-choco-${{ matrix.pkgs }} -- $TARGETS'
[d2074d]155    - name: Prepare logs artifact
156      shell: bash
157      run: |
[f3d331]158        mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a || tar -c --ignore-failed-read -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a)-save.tar" $a ; fi; done; cp -r logs/*  "artifacts/$LOGS_ARTIFACT_NAME"
[d2074d]159      if: always()
160    - uses: actions/upload-artifact@v2
161      with:
162        path: artifacts
163        name: ${{ env.LOGS_ARTIFACT_NAME }}
164      if: always()
165    - name: Print out logs for immediate inspection
166      # The markup in the output is a GitHub Actions logging command
167      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
168      shell: bash
169      run: |
170        find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
171      if: always()
172    - name: Prepare sage-local artifact
173      # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
[68f6c9]174      # We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
[d2074d]175      run: |
[68f6c9]176        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f "${{ env.SAGE_LOCAL }}"/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files "${{ env.SAGE_LOCAL }}"'
[d2074d]177      if: always()
178    - uses: actions/upload-artifact@v2
179      # upload-artifact@v2 does not support whitespace in file names.
180      # so we tar up the directory ourselves
181      with:
182        path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
183        name: ${{ env.LOCAL_ARTIFACT_NAME }}
184      if: always()
185
186  docker:
187    runs-on: ubuntu-latest
188    needs: [dist]
189    strategy:
190      fail-fast: false
191      max-parallel: 32
192      matrix:
[39d022e]193        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-focal-i386, debian-buster-i386, centos-7-i386, raspbian-buster-armhf]
[d2074d]194        tox_packages_factor: [minimal, standard]
195    env:
196      TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
197      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
[653268]198      DOCKER_TARGETS: configured with-targets with-targets-optional
[d2074d]199    steps:
200      - name: Check out SageMath
201        uses: actions/checkout@v2
202        with:
203          repository: ${{ env.SAGE_REPO }}
204          ref: ${{ env.SAGE_REF }}
205        if: env.SAGE_REPO != ''
206      - name: Check out git-trac-command
207        uses: actions/checkout@v2
208        with:
209          repository: sagemath/git-trac-command
210          path: git-trac-command
211        if: env.SAGE_TRAC_GIT != ''
212      - name: Check out SageMath from trac.sagemath.org
213        shell: bash {0}
214        run: |
[efb3fc]215          if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac ${{ env.SAGE_TRAC_COMMAND }} $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
[d2074d]216        if: env.SAGE_TRAC_GIT != ''
217      - uses: actions/download-artifact@v2
218        with:
219          path: upstream
220          name: upstream
221      - name: Install test prerequisites
222        run: |
223          sudo DEBIAN_FRONTEND=noninteractive apt-get update
224          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox python3-setuptools
225      - name: Update Sage packages from upstream artifact
226        run: |
227          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
[1a0966]228      - name: Configure and build Sage distribution within a Docker container
229        run: |
[d2074d]230          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
[1a0966]231      - name: Copy logs from the Docker image or build container
[d2074d]232        run: |
233          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
234          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
235          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
236          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
237        if: always()
238      - uses: actions/upload-artifact@v2
239        with:
240          path: artifacts
241          name: ${{ env.LOGS_ARTIFACT_NAME }}
242        if: always()
243      - name: Print out logs for immediate inspection
244        # and markup the output with GitHub Actions logging commands
245        run: |
246          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
247        if: always()
[1a0966]248      - name: Push Docker images
[d2074d]249        run: |
250          if [ -f .tox/$TOX_ENV/Dockertags ]; then
251            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
252            if [ -z "$TOKEN" ]; then
253              TOKEN="${{ secrets.GITHUB_TOKEN }}"
254            fi
255            echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
256            for a in $(cat .tox/$TOX_ENV/Dockertags); do
257              FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a
258              docker tag $a $FULL_TAG
259              echo Pushing $FULL_TAG
260              docker push $FULL_TAG
261            done || echo "(Ignoring errors)"
262          fi
263        if: always()
264
265  macos:
266
267    runs-on: macos-latest
268    strategy:
269      fail-fast: false
270      max-parallel: 4
271      matrix:
[7d6820]272        os: [ macos-10.15, macos-11.0 ]
[d2074d]273        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos]
274        tox_packages_factor: [minimal, standard]
[7d6820]275        xcode_version_factor: [11.7, default, 12.3]
[d2074d]276
277    needs: [dist]
278
279    env:
280      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
[7d6820]281      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
[c42568]282      DOCKER_TARGETS: configured with-targets with-targets-optional
283
[d2074d]284    steps:
285
[7d6820]286      - name: Select Xcode version
287        run: |
288          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
[d2074d]289      - name: Check out SageMath
290        uses: actions/checkout@v2
291        with:
292          repository: ${{ env.SAGE_REPO }}
293          ref: ${{ env.SAGE_REF }}
294        if: env.SAGE_REPO != ''
295      - name: Check out git-trac-command
296        uses: actions/checkout@v2
297        with:
298          repository: sagemath/git-trac-command
299          path: git-trac-command
300        if: env.SAGE_TRAC_GIT != ''
301      - name: Check out SageMath from trac.sagemath.org
302        shell: bash {0}
303        run: |
[efb3fc]304          if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac ${{ env.SAGE_TRAC_COMMAND }} $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
[d2074d]305        if: env.SAGE_TRAC_GIT != ''
306      - uses: actions/download-artifact@v2
307        with:
308          path: upstream
309          name: upstream
310      - name: Update Sage packages from upstream artifact
311        run: |
312          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff)
313
314      - name: Install test prerequisites
315        run: |
316          brew install tox
317      - name: Build and test with tox
318        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
319        # For doctesting, we use a lower parallelization to avoid timeouts.
320        run: |
321          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
322      - name: Prepare logs artifact
323        run: |
324          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
325        if: always()
326      - uses: actions/upload-artifact@v1
327        with:
328          path: artifacts
329          name: ${{ env.LOGS_ARTIFACT_NAME }}
330        if: always()
331      - name: Print out logs for immediate inspection
332        # and markup the output with GitHub Actions logging commands
333        run: |
334          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
335        if: always()
Note: See TracBrowser for help on using the repository browser.