1 | name: Run Sage CI for Linux/Cygwin/macOS |
---|
2 | |
---|
3 | ## This GitHub Actions workflow provides: |
---|
4 | ## |
---|
5 | ## - portability testing, by building and testing this project on many platforms |
---|
6 | ## (Linux variants and Cygwin), each with two configurations (installed packages), |
---|
7 | ## |
---|
8 | ## - continuous integration, by building and testing other software |
---|
9 | ## that depends on this project. |
---|
10 | ## |
---|
11 | ## It runs on every pull request and push of a tag to the GitHub repository. |
---|
12 | ## |
---|
13 | ## The testing can be monitored in the "Actions" tab of the GitHub repository. |
---|
14 | ## |
---|
15 | ## After all jobs have finished (or are canceled) and a short delay, |
---|
16 | ## tar files of all logs are made available as "build artifacts". |
---|
17 | ## |
---|
18 | ## This GitHub Actions workflow uses the portability testing framework |
---|
19 | ## of SageMath (https://www.sagemath.org/). For more information, see |
---|
20 | ## https://doc.sagemath.org/html/en/developer/portability_testing.html |
---|
21 | |
---|
22 | ## The workflow consists of two jobs: |
---|
23 | ## |
---|
24 | ## - First, it builds a source distribution of the project |
---|
25 | ## and generates a script "update-pkgs.sh". It uploads them |
---|
26 | ## as a build artifact named upstream. |
---|
27 | ## |
---|
28 | ## - Second, it checks out a copy of the SageMath source tree. |
---|
29 | ## It downloads the upstream artifact and replaces the project's |
---|
30 | ## package in the SageMath distribution by the newly packaged one |
---|
31 | ## from the upstream artifact, by running the script "update-pkgs.sh". |
---|
32 | ## Then it builds a small portion of the Sage distribution. |
---|
33 | ## |
---|
34 | ## Many copies of the second step are run in parallel for each of the tested |
---|
35 | ## systems/configurations. |
---|
36 | |
---|
37 | #on: [push, pull_request] |
---|
38 | |
---|
39 | on: |
---|
40 | pull_request: |
---|
41 | types: [opened, synchronize] |
---|
42 | push: |
---|
43 | tags: |
---|
44 | - '*' |
---|
45 | |
---|
46 | env: |
---|
47 | # Ubuntu packages to install so that the project's "make dist" can succeed |
---|
48 | DIST_PREREQ: tar |
---|
49 | # Name of this project in the Sage distribution |
---|
50 | SPKG: singular |
---|
51 | # Sage distribution packages to build |
---|
52 | TARGETS_PRE: build/make/Makefile |
---|
53 | TARGETS: SAGE_CHECK=yes singular |
---|
54 | TARGETS_OPTIONAL: SAGE_CHECK=warn pynac pysingular |
---|
55 | # Standard setting: Test the current beta release of Sage: |
---|
56 | SAGE_REPO: sagemath/sage |
---|
57 | SAGE_REF: develop |
---|
58 | # Temporarily test on the branch from sage ticket for singular 4.1.3 upgrade |
---|
59 | SAGE_TRAC_GIT: git://trac.sagemath.org/sage.git |
---|
60 | SAGE_TICKET: 25993 |
---|
61 | REMOVE_PATCHES: "*" |
---|
62 | |
---|
63 | jobs: |
---|
64 | |
---|
65 | dist: |
---|
66 | runs-on: ubuntu-latest |
---|
67 | steps: |
---|
68 | - name: Check out ${{ env.SPKG }} |
---|
69 | uses: actions/checkout@v2 |
---|
70 | with: |
---|
71 | path: build/pkgs/${{ env.SPKG }}/src |
---|
72 | - name: Install prerequisites |
---|
73 | run: | |
---|
74 | sudo DEBIAN_FRONTEND=noninteractive apt-get update |
---|
75 | sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ |
---|
76 | - name: Run make dist, prepare upstream artifact |
---|
77 | # make dist broken, make_tar.sh broken - https://github.com/Singular/Singular/issues/1016 |
---|
78 | run: | |
---|
79 | (cd build/pkgs/${{ env.SPKG }}/src && (./make_tar.sh || tar cfz singular-tar-so-far.tar.gz singular-4.* ) ) \ |
---|
80 | && mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \ |
---|
81 | && echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=optional" > upstream/update-pkgs.sh \ |
---|
82 | && if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \ |
---|
83 | && ls -l upstream/ |
---|
84 | - uses: actions/upload-artifact@v2 |
---|
85 | with: |
---|
86 | path: upstream |
---|
87 | name: upstream |
---|
88 | |
---|
89 | cygwin: |
---|
90 | env: |
---|
91 | STAGE: i-a |
---|
92 | LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }} |
---|
93 | LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }} |
---|
94 | MAKE: make -j8 |
---|
95 | SAGE_NUM_THREADS: 3 |
---|
96 | SAGE_CHECK: yes |
---|
97 | SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl" |
---|
98 | CYGWIN: winsymlinks:native |
---|
99 | CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url |
---|
100 | SAGE_FAT_BINARY: yes |
---|
101 | |
---|
102 | runs-on: windows-latest |
---|
103 | |
---|
104 | needs: [dist] |
---|
105 | |
---|
106 | strategy: |
---|
107 | fail-fast: false |
---|
108 | matrix: |
---|
109 | pkgs: [minimal, standard] |
---|
110 | steps: |
---|
111 | - run: | |
---|
112 | git config --global core.autocrlf false |
---|
113 | git config --global core.symlinks true |
---|
114 | - name: install cygwin with choco |
---|
115 | shell: bash {0} |
---|
116 | run: | |
---|
117 | choco --version |
---|
118 | choco install git python3 --source cygwin |
---|
119 | - name: Check out SageMath |
---|
120 | uses: actions/checkout@v2 |
---|
121 | with: |
---|
122 | repository: ${{ env.SAGE_REPO }} |
---|
123 | ref: ${{ env.SAGE_REF }} |
---|
124 | if: env.SAGE_REPO != '' |
---|
125 | - name: Check out git-trac-command |
---|
126 | uses: actions/checkout@v2 |
---|
127 | with: |
---|
128 | repository: sagemath/git-trac-command |
---|
129 | path: git-trac-command |
---|
130 | if: env.SAGE_TRAC_GIT != '' |
---|
131 | - name: Check out SageMath from trac.sagemath.org |
---|
132 | shell: bash {0} |
---|
133 | # Random sleep and retry to limit the load on trac.sagemath.org |
---|
134 | run: | |
---|
135 | if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1 |
---|
136 | if: env.SAGE_TRAC_GIT != '' |
---|
137 | - uses: actions/download-artifact@v2 |
---|
138 | with: |
---|
139 | path: upstream |
---|
140 | name: upstream |
---|
141 | - name: install minimal prerequisites with choco |
---|
142 | shell: bash {0} |
---|
143 | run: | |
---|
144 | choco --version |
---|
145 | PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt) |
---|
146 | choco install $PACKAGES --source cygwin |
---|
147 | - name: Update Sage packages from upstream artifact |
---|
148 | run: | |
---|
149 | C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && ls -l upstream/ && export PATH="$(pwd)/build/bin:$PATH:/usr/local/bin:/usr/bin" && (cd upstream && bash -x update-pkgs.sh) && git diff' |
---|
150 | - name: bootstrap |
---|
151 | run: | |
---|
152 | C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap' |
---|
153 | - name: install additional cygwin packages with choco |
---|
154 | if: contains(matrix.pkgs, 'standard') |
---|
155 | shell: bash {0} |
---|
156 | run: | |
---|
157 | PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt) |
---|
158 | choco install $PACKAGES --source cygwin |
---|
159 | - name: configure |
---|
160 | run: | |
---|
161 | C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS' |
---|
162 | - name: make |
---|
163 | run: | |
---|
164 | C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 base-toolchain && make -k -w V=1 $TARGETS' |
---|
165 | - name: make |
---|
166 | run: | |
---|
167 | C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=1 $TARGETS_OPTIONAL' |
---|
168 | - name: Prepare logs artifact |
---|
169 | shell: bash |
---|
170 | run: | |
---|
171 | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a || tar -c --ignore-failed-read -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a)-save.tar" $a ; fi; done; cp -r logs/* "artifacts/$LOGS_ARTIFACT_NAME" |
---|
172 | if: always() |
---|
173 | - uses: actions/upload-artifact@v2 |
---|
174 | with: |
---|
175 | path: artifacts |
---|
176 | name: ${{ env.LOGS_ARTIFACT_NAME }} |
---|
177 | if: always() |
---|
178 | - name: Print out logs for immediate inspection |
---|
179 | # The markup in the output is a GitHub Actions logging command |
---|
180 | # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions |
---|
181 | shell: bash |
---|
182 | run: | |
---|
183 | find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \; |
---|
184 | if: always() |
---|
185 | - name: Prepare sage-local artifact |
---|
186 | # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows. |
---|
187 | # We remove the local/lib64 link, which will be recreated by the next stage. |
---|
188 | run: | |
---|
189 | C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f local/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files local' |
---|
190 | if: always() |
---|
191 | - uses: actions/upload-artifact@v2 |
---|
192 | # upload-artifact@v2 does not support whitespace in file names. |
---|
193 | # so we tar up the directory ourselves |
---|
194 | with: |
---|
195 | path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar |
---|
196 | name: ${{ env.LOCAL_ARTIFACT_NAME }} |
---|
197 | if: always() |
---|
198 | |
---|
199 | docker: |
---|
200 | runs-on: ubuntu-latest |
---|
201 | needs: [dist] |
---|
202 | strategy: |
---|
203 | fail-fast: false |
---|
204 | max-parallel: 32 |
---|
205 | matrix: |
---|
206 | tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386] |
---|
207 | tox_packages_factor: [minimal, standard] |
---|
208 | env: |
---|
209 | TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} |
---|
210 | LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} |
---|
211 | DOCKER_TARGETS: configured with-targets |
---|
212 | steps: |
---|
213 | - name: Check out SageMath |
---|
214 | uses: actions/checkout@v2 |
---|
215 | with: |
---|
216 | repository: ${{ env.SAGE_REPO }} |
---|
217 | ref: ${{ env.SAGE_REF }} |
---|
218 | if: env.SAGE_REPO != '' |
---|
219 | - name: Check out git-trac-command |
---|
220 | uses: actions/checkout@v2 |
---|
221 | with: |
---|
222 | repository: sagemath/git-trac-command |
---|
223 | path: git-trac-command |
---|
224 | if: env.SAGE_TRAC_GIT != '' |
---|
225 | - name: Check out SageMath from trac.sagemath.org |
---|
226 | shell: bash {0} |
---|
227 | run: | |
---|
228 | if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1 |
---|
229 | if: env.SAGE_TRAC_GIT != '' |
---|
230 | - uses: actions/download-artifact@v2 |
---|
231 | with: |
---|
232 | path: upstream |
---|
233 | name: upstream |
---|
234 | - name: Install test prerequisites |
---|
235 | run: | |
---|
236 | sudo DEBIAN_FRONTEND=noninteractive apt-get update |
---|
237 | sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox python3-setuptools |
---|
238 | - name: Update Sage packages from upstream artifact |
---|
239 | run: | |
---|
240 | (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff) |
---|
241 | - name: Configure and build Sage distribution within a Docker container |
---|
242 | run: | |
---|
243 | set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" |
---|
244 | - name: Copy logs from the Docker image or build container |
---|
245 | run: | |
---|
246 | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" |
---|
247 | cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME" |
---|
248 | if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi |
---|
249 | if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi |
---|
250 | if: always() |
---|
251 | - uses: actions/upload-artifact@v2 |
---|
252 | with: |
---|
253 | path: artifacts |
---|
254 | name: ${{ env.LOGS_ARTIFACT_NAME }} |
---|
255 | if: always() |
---|
256 | - name: Print out logs for immediate inspection |
---|
257 | # and markup the output with GitHub Actions logging commands |
---|
258 | run: | |
---|
259 | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" |
---|
260 | if: always() |
---|
261 | - name: Push Docker images |
---|
262 | run: | |
---|
263 | if [ -f .tox/$TOX_ENV/Dockertags ]; then |
---|
264 | TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" |
---|
265 | if [ -z "$TOKEN" ]; then |
---|
266 | TOKEN="${{ secrets.GITHUB_TOKEN }}" |
---|
267 | fi |
---|
268 | echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin |
---|
269 | for a in $(cat .tox/$TOX_ENV/Dockertags); do |
---|
270 | FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a |
---|
271 | docker tag $a $FULL_TAG |
---|
272 | echo Pushing $FULL_TAG |
---|
273 | docker push $FULL_TAG |
---|
274 | done || echo "(Ignoring errors)" |
---|
275 | fi |
---|
276 | if: always() |
---|
277 | |
---|
278 | macos: |
---|
279 | |
---|
280 | runs-on: macos-latest |
---|
281 | strategy: |
---|
282 | fail-fast: false |
---|
283 | max-parallel: 4 |
---|
284 | matrix: |
---|
285 | tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos] |
---|
286 | tox_packages_factor: [minimal, standard] |
---|
287 | |
---|
288 | needs: [dist] |
---|
289 | |
---|
290 | env: |
---|
291 | TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} |
---|
292 | LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} |
---|
293 | steps: |
---|
294 | |
---|
295 | - name: Check out SageMath |
---|
296 | uses: actions/checkout@v2 |
---|
297 | with: |
---|
298 | repository: ${{ env.SAGE_REPO }} |
---|
299 | ref: ${{ env.SAGE_REF }} |
---|
300 | if: env.SAGE_REPO != '' |
---|
301 | - name: Check out git-trac-command |
---|
302 | uses: actions/checkout@v2 |
---|
303 | with: |
---|
304 | repository: sagemath/git-trac-command |
---|
305 | path: git-trac-command |
---|
306 | if: env.SAGE_TRAC_GIT != '' |
---|
307 | - name: Check out SageMath from trac.sagemath.org |
---|
308 | shell: bash {0} |
---|
309 | run: | |
---|
310 | if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac checkout $SAGE_TICKET; then exit 0; fi; sleep 40; done; exit 1 |
---|
311 | if: env.SAGE_TRAC_GIT != '' |
---|
312 | - uses: actions/download-artifact@v2 |
---|
313 | with: |
---|
314 | path: upstream |
---|
315 | name: upstream |
---|
316 | - name: Update Sage packages from upstream artifact |
---|
317 | run: | |
---|
318 | (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff) |
---|
319 | |
---|
320 | - name: Install test prerequisites |
---|
321 | run: | |
---|
322 | brew install tox |
---|
323 | - name: Build and test with tox |
---|
324 | # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts. |
---|
325 | # For doctesting, we use a lower parallelization to avoid timeouts. |
---|
326 | run: | |
---|
327 | MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS |
---|
328 | - name: Prepare logs artifact |
---|
329 | run: | |
---|
330 | mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME" |
---|
331 | if: always() |
---|
332 | - uses: actions/upload-artifact@v1 |
---|
333 | with: |
---|
334 | path: artifacts |
---|
335 | name: ${{ env.LOGS_ARTIFACT_NAME }} |
---|
336 | if: always() |
---|
337 | - name: Print out logs for immediate inspection |
---|
338 | # and markup the output with GitHub Actions logging commands |
---|
339 | run: | |
---|
340 | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" |
---|
341 | if: always() |
---|