From d621b2ceed23f4ecc945b00ffd6e859b70b89b50 Mon Sep 17 00:00:00 2001
From: Allard Hendriksen <allard@allard-dell-xps-9560>
Date: Tue, 17 Sep 2019 14:43:58 +0200
Subject: Use conda compiler toolchain for conda builds
For both libastra and astra-toolbox:
1) We do not use script_env to set CC/CXX anymore, since the compilers
are installed by conda.
2) The build string is made useful by including either the python+numpy
version or the cudatoolkit version that the package was built with.
3) Some clean-up of build.sh in buildenv/
For libastra:
1) The libastra.so is built with the conda C/C++ compiler
toolchain. This has two benefits:
1) The rpath of libastra.so is set to $ORIGIN, which makes linking
easier for dependent packages.
2) libastra.so is linkable against ancient versions of glibc. With
old versions of memcpy.
2) The C/C++ compiler version is fixed to 5.4.0
3) In libastra/build.sh, we rename $CONDA_PREFIX to
$PREFIX. Apparently, this is how it is supposed to be done. For me,
$CONDA_PREFIX was suddenly undefined. Why this was not a problem
before, is unclear to me.
4) The cudatoolkit runtime dependency is pinned with pin_compatible
5) The libastra conda package now provides headers and .pc file. This
is useful for building C++ packages that depend on astra.
6) Remove some old code related to cudatoolkit<8.0.
For astra-toolbox:
1) astra-toolbox uses the conda-provided compilers
2) The compilers are fixed to version 7.3
3) Add boost to host requirements of astra-toolbox
Notes on testing:
- The libastra build has been tested with all versions of cudatoolkit
- The astra-toolbox build has been tested with all provided versions
of python after building a single cudatoolkit version of libastra.
How to test this branch:
- It should work by just editing
`python/conda/linux_release/buildenv/build.sh`. Set
BRANCH=CI-use-conda-c-compiler-toolchain
URL=https://github.com/ahendriksen/astra-toolbox
and run release.sh from the `python/conda/linux_release` directory.
---
.../astra-toolbox/conda_build_config_py27.yaml | 5 +++++
.../astra-toolbox/conda_build_config_py35.yaml | 4 ++++
.../astra-toolbox/conda_build_config_py36.yaml | 4 ++++
.../astra-toolbox/conda_build_config_py37.yaml | 4 ++++
python/conda/astra-toolbox/meta.yaml | 8 ++++---
python/conda/libastra/build.sh | 26 ++++++++--------------
python/conda/libastra/linux_build_config.yaml | 4 ++++
python/conda/libastra/meta.yaml | 13 ++++++-----
python/conda/linux_release/buildenv/build.sh | 13 ++++++++---
9 files changed, 52 insertions(+), 29 deletions(-)
mode change 100644 => 100755 python/conda/libastra/build.sh
mode change 100644 => 100755 python/conda/linux_release/buildenv/build.sh
diff --git a/python/conda/astra-toolbox/conda_build_config_py27.yaml b/python/conda/astra-toolbox/conda_build_config_py27.yaml
index f119b61..869d22e 100644
--- a/python/conda/astra-toolbox/conda_build_config_py27.yaml
+++ b/python/conda/astra-toolbox/conda_build_config_py27.yaml
@@ -8,3 +8,8 @@ numpy:
- 1.14
- 1.15
- 1.16
+
+c_compiler_version:
+ - 7.3
+cxx_compiler_version:
+ - 7.3
diff --git a/python/conda/astra-toolbox/conda_build_config_py35.yaml b/python/conda/astra-toolbox/conda_build_config_py35.yaml
index 8dd963c..72d4cef 100644
--- a/python/conda/astra-toolbox/conda_build_config_py35.yaml
+++ b/python/conda/astra-toolbox/conda_build_config_py35.yaml
@@ -3,3 +3,7 @@ python:
numpy:
- 1.15
+c_compiler_version:
+ - 7.3
+cxx_compiler_version:
+ - 7.3
diff --git a/python/conda/astra-toolbox/conda_build_config_py36.yaml b/python/conda/astra-toolbox/conda_build_config_py36.yaml
index f043435..996e22c 100644
--- a/python/conda/astra-toolbox/conda_build_config_py36.yaml
+++ b/python/conda/astra-toolbox/conda_build_config_py36.yaml
@@ -5,3 +5,7 @@ numpy:
- 1.14
- 1.15
- 1.16
+c_compiler_version:
+ - 7.3
+cxx_compiler_version:
+ - 7.3
diff --git a/python/conda/astra-toolbox/conda_build_config_py37.yaml b/python/conda/astra-toolbox/conda_build_config_py37.yaml
index ebbf44a..4282407 100644
--- a/python/conda/astra-toolbox/conda_build_config_py37.yaml
+++ b/python/conda/astra-toolbox/conda_build_config_py37.yaml
@@ -6,3 +6,7 @@ numpy:
- 1.14
- 1.15
- 1.16
+c_compiler_version:
+ - 7.3
+cxx_compiler_version:
+ - 7.3
diff --git a/python/conda/astra-toolbox/meta.yaml b/python/conda/astra-toolbox/meta.yaml
index 054751b..b3c036b 100644
--- a/python/conda/astra-toolbox/meta.yaml
+++ b/python/conda/astra-toolbox/meta.yaml
@@ -8,8 +8,7 @@ source:
build:
number: 0
- script_env:
- - CC # [linux]
+ string: py_{{ python }}_numpy_{{ numpy }}
test:
imports:
@@ -24,14 +23,17 @@ test:
requirements:
build:
+ - {{ compiler('c') }} # [win or linux]
+ - {{ compiler('cxx') }} # [linux]
+ host:
- python
- cython >=0.13
+ - boost # [osx or linux]
- nomkl # [not win]
- numpy {{ numpy }}
- scipy
- six
- libastra ==1.9.0.dev11
-
run:
- python
- {{ pin_compatible('numpy', min_pin='x.x', max_pin='x.x') }}
diff --git a/python/conda/libastra/build.sh b/python/conda/libastra/build.sh
old mode 100644
new mode 100755
index d19c7af..4e48ba6
--- a/python/conda/libastra/build.sh
+++ b/python/conda/libastra/build.sh
@@ -18,35 +18,27 @@ cd $SRC_DIR/build/linux
$SRC_DIR/build/linux/autogen.sh
-# Add C++11 to compiler flags if nvcc supports it, mostly to work around a boost bug
+# Add C++11 to compiler flags, mostly to work around a boost bug.
+# Since we require cudatoolkit>=8.0 nvcc supports this.
NVCC=$CUDA_ROOT/bin/nvcc
-echo "int main(){return 0;}" > $CONDA_PREFIX/test.cu
-$NVCC $CONDA_PREFIX/test.cu -ccbin $CC --std=c++11 -o $CONDA_PREFIX/test.out > /dev/null 2>&1 && EXTRA_NVCCFLAGS="--std=c++11" || true
-rm -f $CONDA_PREFIX/test.out $CONDA_PREFIX/test.cu
+EXTRA_NVCCFLAGS="--std=c++11"
-$SRC_DIR/build/linux/configure --with-install-type=prefix --with-cuda=$CUDA_ROOT --prefix=$CONDA_PREFIX NVCCFLAGS="-ccbin $CC -I$CONDA_PREFIX/include $EXTRA_NVCCFLAGS" CC=$CC CXX=$CXX CPPFLAGS="-I$CONDA_PREFIX/include"
+
+$SRC_DIR/build/linux/configure --with-install-type=prefix --with-cuda=$CUDA_ROOT --prefix=$PREFIX NVCCFLAGS="-ccbin $CC -I$PREFIX/include $EXTRA_NVCCFLAGS" CC=$CC CXX=$CXX CPPFLAGS="-I$PREFIX/include"
# Clean, because we may be re-using this source tree when building
# multiple variants of this conda package.
make clean
-make -j $CPU_COUNT install-libraries
+make -j $CPU_COUNT
+make -j $CPU_COUNT install-dev
test -d $CUDA_ROOT/lib64 && LIBPATH="$CUDA_ROOT/lib64" || LIBPATH="$CUDA_ROOT/lib"
case `uname` in
Darwin*)
- cp -P $LIBPATH/libcudart.*.dylib $CONDA_PREFIX/lib
- cp -P $LIBPATH/libcufft.*.dylib $CONDA_PREFIX/lib
- ;;
- Linux*)
- if [ "$cudatoolkit" = "7.0" ]; then
- # For some reason conda-build adds these symlinks automatically for
- # cudatoolkit-5.5 and 6.0, but not 7.0. For 7.5 these symlinks are not
- # necessary, and for 8.0 the cudatoolkit packages includes them.
- ln -T -s libcudart.so.7.0.28 $CONDA_PREFIX/lib/libcudart.so.7.0
- ln -T -s libcufft.so.7.0.35 $CONDA_PREFIX/lib/libcufft.so.7.0
- fi
+ cp -P $LIBPATH/libcudart.*.dylib $PREFIX/lib
+ cp -P $LIBPATH/libcufft.*.dylib $PREFIX/lib
;;
esac
diff --git a/python/conda/libastra/linux_build_config.yaml b/python/conda/libastra/linux_build_config.yaml
index 41db828..b415b4a 100644
--- a/python/conda/libastra/linux_build_config.yaml
+++ b/python/conda/libastra/linux_build_config.yaml
@@ -4,3 +4,7 @@ cudatoolkit:
- 9.2
- 10.0
- 10.1
+c_compiler_version: # [linux or win]
+ - 5.4 # [linux or win]
+cxx_compiler_version: # [linux or win]
+ - 5.4 # [linux or win]
diff --git a/python/conda/libastra/meta.yaml b/python/conda/libastra/meta.yaml
index 59c3a09..407929b 100644
--- a/python/conda/libastra/meta.yaml
+++ b/python/conda/libastra/meta.yaml
@@ -8,21 +8,22 @@ source:
build:
number: 0
- script_env:
- - CC # [linux]
- - CXX # [linux]
+ string: cuda_{{ cudatoolkit }} # [linux]
requirements:
build:
- - {{compiler('c')}} # [win]
+ - {{ compiler('c') }} # [win or linux]
+ - {{ compiler('cxx') }} # [linux]
- boost # [osx]
- automake # [osx]
- autoconf # [osx]
- libtool # [osx]
+ host:
+ - boost # [osx or linux]
- cudatoolkit {{ cudatoolkit }} # [linux]
-
run:
- - cudatoolkit {{ cudatoolkit }} # [linux]
+ # See: https://github.com/conda-forge/conda-forge.github.io/issues/687#issuecomment-460095230
+ - {{ pin_compatible('cudatoolkit', max_pin='x.x') }} # [linux]
about:
home: http://www.astra-toolbox.com
diff --git a/python/conda/linux_release/buildenv/build.sh b/python/conda/linux_release/buildenv/build.sh
old mode 100644
new mode 100755
index cc6dc31..158c1d5
--- a/python/conda/linux_release/buildenv/build.sh
+++ b/python/conda/linux_release/buildenv/build.sh
@@ -1,16 +1,23 @@
#!/bin/sh
-export CC=gcc
-export CXX=g++
+set -e
+
+BRANCH=master
+URL=https://github.com/astra-toolbox/astra-toolbox
+
+echo "Cloning from ${URL}"
+echo " branch: ${BRANCH}"
cd /root
-git clone --depth 1 --branch master https://github.com/astra-toolbox/astra-toolbox
+git clone --depth 1 --branch ${BRANCH} ${URL}
+
[ $# -eq 0 ] || perl -pi -e "s/^(\s*version:\s*)[0-9a-z+\.']+$/\${1}'$1'/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml
[ $# -eq 0 ] || perl -pi -e "s/^(\s*number:\s*)[0-9]+$/\${1}$2/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml
[ $# -eq 0 ] || perl -pi -e "s/^(\s*-\s*libastra\s*==\s*)[0-9a-z+\.]+$/\${1}$1/" astra-toolbox/python/conda/astra-toolbox/meta.yaml
conda-build -m astra-toolbox/python/conda/libastra/linux_build_config.yaml astra-toolbox/python/conda/libastra
+
for i in 27 35 36 37; do
conda-build -m astra-toolbox/python/conda/astra-toolbox/conda_build_config_py$i.yaml astra-toolbox/python/conda/astra-toolbox
done
--
cgit v1.2.3