source: git/.github/workflows/ci-sage.yml @ ee5b48

fieker-DuValspielwiese
Last change on this file since ee5b48 was ee5b48, checked in by Matthias Koeppe <mkoeppe@…>, 4 years ago
'make dist' is broken, use ./make_tar.sh instead
  • Property mode set to 100644
File size: 15.3 KB
Line 
1name: Run Sage CI for Linux/Cygwin/macOS
2
3## This GitHub Actions workflow provides:
4##
5##  - portability testing, by building and testing this project on many platforms
6##    (Linux variants and Cygwin), each with two configurations (installed packages),
7##
8##  - continuous integration, by building and testing other software
9##    that depends on this project.
10##
11## It runs on every pull request and push of a tag to the GitHub repository.
12##
13## The testing can be monitored in the "Actions" tab of the GitHub repository.
14##
15## After all jobs have finished (or are canceled) and a short delay,
16## tar files of all logs are made available as "build artifacts".
17##
18## This GitHub Actions workflow uses the portability testing framework
19## of SageMath (https://www.sagemath.org/).  For more information, see
20## https://doc.sagemath.org/html/en/developer/portability_testing.html
21
22## The workflow consists of two jobs:
23##
24##  - First, it builds a source distribution of the project
25##    and generates a script "update-pkgs.sh".  It uploads them
26##    as a build artifact named upstream.
27##
28##  - Second, it checks out a copy of the SageMath source tree.
29##    It downloads the upstream artifact and replaces the project's
30##    package in the SageMath distribution by the newly packaged one
31##    from the upstream artifact, by running the script "update-pkgs.sh".
32##    Then it builds a small portion of the Sage distribution.
33##
34## Many copies of the second step are run in parallel for each of the tested
35## systems/configurations.
36
37#on: [push, pull_request]
38
39on:
40  pull_request:
41    types: [opened, synchronize]
42  push:
43    tags:
44      - '*'
45
46env:
47  # Ubuntu packages to install so that the project's "make dist" can succeed
48  DIST_PREREQ: tar
49  # Name of this project in the Sage distribution
50  SPKG:        singular
51  # Sage distribution packages to build
52  TARGETS_PRE: build/make/Makefile
53  TARGETS:     SAGE_CHECK=yes singular
54  TARGETS_OPTIONAL: SAGE_CHECK=warn pynac pysingular
55  # Standard setting: Test the current beta release of Sage:
56  SAGE_REPO:   sagemath/sage
57  SAGE_REF:    develop
58  # Temporarily test on the branch from sage ticket for singular 4.1.3 upgrade
59  SAGE_TRAC_GIT: git://trac.sagemath.org/sage.git
60  SAGE_TICKET: 25993
61  REMOVE_PATCHES: "*"
62
63jobs:
64
65  dist:
66    runs-on: ubuntu-latest
67    steps:
68      - name: Check out ${{ env.SPKG }}
69        uses: actions/checkout@v2
70        with:
71          path: build/pkgs/${{ env.SPKG }}/src
72      - name: Install prerequisites
73        run: |
74          sudo DEBIAN_FRONTEND=noninteractive apt-get update
75          sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ
76      - name: Run make dist, prepare upstream artifact
77        run: |
78          (cd build/pkgs/${{ env.SPKG }}/src && ./make_tar.sh) \
79          && mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \
80          && echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=optional" > upstream/update-pkgs.sh \
81          && if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \
82          && ls -l upstream/
83      - uses: actions/upload-artifact@v2
84        with:
85          path: upstream
86          name: upstream
87
88  cygwin:
89    env:
90      STAGE: i-a
91      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
92      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
93      MAKE: make -j8
94      SAGE_NUM_THREADS: 3
95      SAGE_CHECK: yes
96      SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
97      CYGWIN: winsymlinks:native
98      CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
99      SAGE_FAT_BINARY: yes
100
101    runs-on: windows-latest
102
103    needs: [dist]
104
105    strategy:
106      fail-fast: false
107      matrix:
108        pkgs: [minimal, standard]
109    steps:
110    - run: |
111        git config --global core.autocrlf false
112        git config --global core.symlinks true
113    - name: install cygwin with choco
114      shell: bash {0}
115      run: |
116        choco --version
117        choco install git python3 --source cygwin
118    - name: Check out SageMath
119      uses: actions/checkout@v2
120      with:
121        repository: ${{ env.SAGE_REPO }}
122        ref: ${{ env.SAGE_REF }}
123      if: env.SAGE_REPO != ''
124    - name: Check out git-trac-command
125      uses: actions/checkout@v2
126      with:
127        repository: sagemath/git-trac-command
128        path: git-trac-command
129      if: env.SAGE_TRAC_GIT != ''
130    - name: Check out SageMath from trac.sagemath.org
131      shell: bash {0}
132      # Random sleep and retry to limit the load on trac.sagemath.org
133      run: |
134        if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
135      if: env.SAGE_TRAC_GIT != ''
136    - uses: actions/download-artifact@v2
137      with:
138        path: upstream
139        name: upstream
140    - name: install minimal prerequisites with choco
141      shell: bash {0}
142      run: |
143        choco --version
144        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
145        choco install $PACKAGES --source cygwin
146    - name: Update Sage packages from upstream artifact
147      run: |
148        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && ls -l upstream/ && export PATH="$(pwd)/build/bin:$PATH:/usr/local/bin:/usr/bin" && (cd upstream && bash -x update-pkgs.sh) && git diff'
149    - name: bootstrap
150      run: |
151        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap'
152    - name: install additional cygwin packages with choco
153      if: contains(matrix.pkgs, 'standard')
154      shell: bash {0}
155      run: |
156        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt)
157        choco install $PACKAGES --source cygwin
158    - name: configure
159      run: |
160        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS'
161    - name: make
162      run: |
163        C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 base-toolchain && make -k -w V=1 $TARGETS'
164    - name: make
165      run: |
166        C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=1 $TARGETS_OPTIONAL'
167    - name: Prepare logs artifact
168      shell: bash
169      run: |
170        mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a || tar -c --ignore-failed-read -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a)-save.tar" $a ; fi; done; cp -r logs/*  "artifacts/$LOGS_ARTIFACT_NAME"
171      if: always()
172    - uses: actions/upload-artifact@v2
173      with:
174        path: artifacts
175        name: ${{ env.LOGS_ARTIFACT_NAME }}
176      if: always()
177    - name: Print out logs for immediate inspection
178      # The markup in the output is a GitHub Actions logging command
179      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
180      shell: bash
181      run: |
182        find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
183      if: always()
184    - name: Prepare sage-local artifact
185      # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
186      # We remove the local/lib64 link, which will be recreated by the next stage.
187      run: |
188        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f local/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files local'
189      if: always()
190    - uses: actions/upload-artifact@v2
191      # upload-artifact@v2 does not support whitespace in file names.
192      # so we tar up the directory ourselves
193      with:
194        path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
195        name: ${{ env.LOCAL_ARTIFACT_NAME }}
196      if: always()
197
198  docker:
199    runs-on: ubuntu-latest
200    needs: [dist]
201    strategy:
202      fail-fast: false
203      max-parallel: 32
204      matrix:
205        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
206        tox_packages_factor: [minimal, standard]
207    env:
208      TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
209      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
210      DOCKER_TARGETS: configured with-targets
211    steps:
212      - name: Check out SageMath
213        uses: actions/checkout@v2
214        with:
215          repository: ${{ env.SAGE_REPO }}
216          ref: ${{ env.SAGE_REF }}
217        if: env.SAGE_REPO != ''
218      - name: Check out git-trac-command
219        uses: actions/checkout@v2
220        with:
221          repository: sagemath/git-trac-command
222          path: git-trac-command
223        if: env.SAGE_TRAC_GIT != ''
224      - name: Check out SageMath from trac.sagemath.org
225        shell: bash {0}
226        run: |
227          if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
228        if: env.SAGE_TRAC_GIT != ''
229      - uses: actions/download-artifact@v2
230        with:
231          path: upstream
232          name: upstream
233      - name: Install test prerequisites
234        run: |
235          sudo DEBIAN_FRONTEND=noninteractive apt-get update
236          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox python3-setuptools
237      - name: Update Sage packages from upstream artifact
238        run: |
239          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
240      - run: |
241          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
242      - name: Copy logs from the docker image or build container
243        run: |
244          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
245          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
246          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
247          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
248        if: always()
249      - uses: actions/upload-artifact@v2
250        with:
251          path: artifacts
252          name: ${{ env.LOGS_ARTIFACT_NAME }}
253        if: always()
254      - name: Print out logs for immediate inspection
255        # and markup the output with GitHub Actions logging commands
256        run: |
257          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
258        if: always()
259      - name: Push docker images
260        run: |
261          if [ -f .tox/$TOX_ENV/Dockertags ]; then
262            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
263            if [ -z "$TOKEN" ]; then
264              TOKEN="${{ secrets.GITHUB_TOKEN }}"
265            fi
266            echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
267            for a in $(cat .tox/$TOX_ENV/Dockertags); do
268              FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a
269              docker tag $a $FULL_TAG
270              echo Pushing $FULL_TAG
271              docker push $FULL_TAG
272            done || echo "(Ignoring errors)"
273          fi
274        if: always()
275
276  macos:
277
278    runs-on: macos-latest
279    strategy:
280      fail-fast: false
281      max-parallel: 4
282      matrix:
283        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos]
284        tox_packages_factor: [minimal, standard]
285
286    needs: [dist]
287
288    env:
289      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
290      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
291    steps:
292
293      - name: Check out SageMath
294        uses: actions/checkout@v2
295        with:
296          repository: ${{ env.SAGE_REPO }}
297          ref: ${{ env.SAGE_REF }}
298        if: env.SAGE_REPO != ''
299      - name: Check out git-trac-command
300        uses: actions/checkout@v2
301        with:
302          repository: sagemath/git-trac-command
303          path: git-trac-command
304        if: env.SAGE_TRAC_GIT != ''
305      - name: Check out SageMath from trac.sagemath.org
306        shell: bash {0}
307        run: |
308          if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1
309        if: env.SAGE_TRAC_GIT != ''
310      - uses: actions/download-artifact@v2
311        with:
312          path: upstream
313          name: upstream
314      - name: Update Sage packages from upstream artifact
315        run: |
316          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff)
317
318      - name: Install test prerequisites
319        run: |
320          brew install tox
321      - name: Build and test with tox
322        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
323        # For doctesting, we use a lower parallelization to avoid timeouts.
324        run: |
325          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
326      - name: Prepare logs artifact
327        run: |
328          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
329        if: always()
330      - uses: actions/upload-artifact@v1
331        with:
332          path: artifacts
333          name: ${{ env.LOGS_ARTIFACT_NAME }}
334        if: always()
335      - name: Print out logs for immediate inspection
336        # and markup the output with GitHub Actions logging commands
337        run: |
338          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
339        if: always()
Note: See TracBrowser for help on using the repository browser.