From c3ad4183964e2f0c59573412fea18251a96ab3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= <remi.cresson@inrae.fr> Date: Wed, 27 Dec 2023 16:30:09 +0100 Subject: [PATCH 1/7] WIP: TF2.14, OTB9 --- Dockerfile | 19 +++---------------- tools/docker/build-deps-gui.txt | 13 ------------- tools/docker/build-env-tf.sh | 1 + 3 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 tools/docker/build-deps-gui.txt diff --git a/Dockerfile b/Dockerfile index 8d1ee2ab..04aea0b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/loc # Upgrade pip RUN pip install --no-cache-dir pip --upgrade # NumPy version is conflicting with system's gdal dep and may require venv -ARG NUMPY_SPEC="==1.22.*" +#ARG NUMPY_SPEC="==1.22.*" # This is to avoid https://github.com/tensorflow/tensorflow/issues/61551 ARG PROTO_SPEC="==4.23.*" RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \ @@ -45,7 +45,7 @@ RUN git config --global advice.detachedHead false ### TF -ARG TF=v2.12.0 +ARG TF=v2.14.0 ARG TENSORRT # Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default) @@ -98,7 +98,7 @@ RUN cd tensorflow \ ### OTB ARG GUI=false -ARG OTB=d74ab47d4308591db4ed5a5ea3b820cef73a39fe +ARG OTB=release-9.0 ARG OTBTESTS=false RUN mkdir /src/otb @@ -111,18 +111,6 @@ RUN apt-get update -y \ && update-ca-certificates \ && git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \ && cd otb && git checkout $OTB \ -# <---------------------------------------- Begin dirty hack -# This is a dirty hack for release 4.0.0alpha -# We have to wait that OTB moves from C++14 to C++17 -# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338 - && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \ - && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt \ - && echo "" > Modules/Segmentation/Conversion/test/CMakeLists.txt \ - && echo "" > Modules/Radiometry/Indices/test/CMakeLists.txt \ - && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \ - && echo "" > Modules/Feature/Edge/test/CMakeLists.txt \ - && echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \ -# <---------------------------------------- End dirty hack && cd .. \ && mkdir -p build \ && cd build \ @@ -206,4 +194,3 @@ RUN python -c "import tensorflow" RUN python -c "import otbtf, tricks" RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')" RUN python -c "from osgeo import gdal" - diff --git a/tools/docker/build-deps-gui.txt b/tools/docker/build-deps-gui.txt deleted file mode 100644 index 1f1f6cf9..00000000 --- a/tools/docker/build-deps-gui.txt +++ /dev/null @@ -1,13 +0,0 @@ -freeglut3-dev -libglew-dev -libglfw3-dev -libqt5opengl5-dev -libqwt-qt5-dev -libx11-dev -libgl-dev -libxmu-dev -libxi-dev -qtbase5-dev -qttools5-dev -qttools5-dev-tools -xvfb diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh index 4eb7c7ff..94546bfe 100644 --- a/tools/docker/build-env-tf.sh +++ b/tools/docker/build-env-tf.sh @@ -20,6 +20,7 @@ export TF_NEED_OPENCL=0 export TF_NEED_OPENCL_SYCL=0 export TF_NEED_VERBS=0 export TF_SET_ANDROID_WORKSPACE=0 +export TF_NEED_CLANG=0 # For MKL support BZL_CONFIGS+=" --config=mkl" #export TF_DOWNLOAD_MKL=1 #export TF_NEED_MKL=0 -- GitLab From 74b6f51b46d1b70e5c3cb5d9edba33a7bebcf72a Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 2 Jan 2024 19:29:42 +0100 Subject: [PATCH 2/7] COMP: remove gui completely from dockerfile --- Dockerfile | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04aea0b8..904431e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,23 +10,15 @@ WORKDIR /tmp ### System packages COPY tools/docker/build-deps-*.txt ./ ARG DEBIAN_FRONTEND=noninteractive -# CLI RUN apt-get update -y && apt-get upgrade -y \ && cat build-deps-cli.txt | xargs apt-get install --no-install-recommends -y \ && apt-get clean && rm -rf /var/lib/apt/lists/* -# Optional GUI -ARG GUI=false -RUN if $GUI; then \ - apt-get update -y \ - && cat build-deps-gui.txt | xargs apt-get install --no-install-recommends -y \ - && apt-get clean && rm -rf /var/lib/apt/lists/* ; fi - ### Python3 links and pip packages RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip # Upgrade pip RUN pip install --no-cache-dir pip --upgrade -# NumPy version is conflicting with system's gdal dep and may require venv -#ARG NUMPY_SPEC="==1.22.*" +# In case NumPy version is conflicting with system's gdal dep and may require venv +ARG NUMPY_SPEC="" # This is to avoid https://github.com/tensorflow/tensorflow/issues/61551 ARG PROTO_SPEC="==4.23.*" RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \ @@ -35,7 +27,7 @@ RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$N # ---------------------------------------------------------------------------- # Tmp builder stage - dangling cache should persist until "docker builder prune" FROM otbtf-base AS builder -# A smaller value may be required to avoid OOM errors when building OTB GUI +# A smaller value may be required to avoid OOM errors when building OTB ARG CPU_RATIO=1 RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib/python3 @@ -79,6 +71,7 @@ RUN cd tensorflow \ && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' # Installation +RUN apt update && apt install -y patchelf RUN cd tensorflow \ && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \ && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \ @@ -97,7 +90,6 @@ RUN cd tensorflow \ ### OTB -ARG GUI=false ARG OTB=release-9.0 ARG OTBTESTS=false @@ -110,15 +102,26 @@ RUN apt-get update -y \ && apt-get install --reinstall ca-certificates -y \ && update-ca-certificates \ && git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \ - && cd otb && git checkout $OTB \ + && cd otb && git checkout $OTB + +# <---------------------------------------- Begin dirty hack +# This is a dirty hack for release 4.0.0alpha +# We have to wait that OTB moves from C++14 to C++17 +# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338 +RUN cd /src/otb/otb \ + && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \ + && echo "" > Modules/Core/ImageManipulation/test/CMakeLists.txt \ + && echo "" > Modules/Core/Conversion/test/CMakeLists.txt \ + && echo "" > Modules/Core/Indices/test/CMakeLists.txt \ + && echo "" > Modules/Core/Edge/test/CMakeLists.txt \ + && echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \ + && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \ +# <---------------------------------------- End dirty hack && cd .. \ && mkdir -p build \ && cd build \ && if $OTBTESTS; then \ echo "-DBUILD_TESTING=ON" >> ../build-flags-otb.txt; fi \ - # Set GL/Qt build flags - && if $GUI; then \ - sed -i -r "s/-DOTB_USE_(QT|OPENGL|GL[UFE][WT])=OFF/-DOTB_USE_\1=ON/" ../build-flags-otb.txt; fi \ # Possible ENH: superbuild-all-dependencies switch, with separated build-deps-minimal.txt and build-deps-otbcli.txt) #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/ " ../build-flags-otb.txt; fi \ && OTB_FLAGS=$(cat "../build-flags-otb.txt") \ @@ -127,7 +130,6 @@ RUN apt-get update -y \ ### OTBTF - copy (without .git/) or clone repository COPY . /src/otbtf -#RUN git clone https://github.com/remicres/otbtf.git /src/otbtf RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf # Rebuild OTB with module @@ -140,12 +142,10 @@ RUN cd /src/otb/build/OTB/build \ -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \ -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \ -Dtensorflow_include_dir=/opt/otbtf/include/tf \ - # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel -DTENSORFLOW_CC_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_cc.so.2 \ -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_framework.so.2 \ && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \ # Cleaning - && ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \ && ( $KEEP_SRC_OTB || rm -rf /src/otb ) \ && rm -rf /root/.cache /tmp/* -- GitLab From 42a9da9902ae9756dd5b6928daadd1e063661fca Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 2 Jan 2024 19:30:03 +0100 Subject: [PATCH 3/7] COMP: update OTB build flags wrt version 9.0.0 --- tools/docker/build-flags-otb.txt | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt index 95b8462c..8c9b0123 100644 --- a/tools/docker/build-flags-otb.txt +++ b/tools/docker/build-flags-otb.txt @@ -1,19 +1,8 @@ --DUSE_SYSTEM_BOOST=ON --DUSE_SYSTEM_CURL=ON --DUSE_SYSTEM_EXPAT=ON --DUSE_SYSTEM_FFTW=ON --DUSE_SYSTEM_FREETYPE=ON --DUSE_SYSTEM_GDAL=OFF --DUSE_SYSTEM_GEOS=ON --DUSE_SYSTEM_GEOTIFF=OFF --DUSE_SYSTEM_GSL=ON --DUSE_SYSTEM_ITK=ON --DUSE_SYSTEM_LIBKML=ON --DUSE_SYSTEM_LIBSVM=ON --DUSE_SYSTEM_MUPARSER=ON --DUSE_SYSTEM_MUPARSERX=ON --DUSE_SYSTEM_OPENCV=ON --DUSE_SYSTEM_PNG=ON --DUSE_SYSTEM_TINYXML=ON --DUSE_SYSTEM_ZLIB=ON --DUSE_SYSTEM_SWIG=ON +-DOTB_BUILD_FeaturesExtraction=ON +-DOTB_BUILD_Hyperspectral=ON +-DOTB_BUILD_Learning=ON +-DOTB_BUILD_Miscellaneous=ON +-DOTB_BUILD_RemoteModules=ON +-DOTB_BUILD_SAR=ON +-DOTB_BUILD_Segmentation=ON +-DOTB_BUILD_StereoProcessing=ON -- GitLab From a3c5c1ba3bdce3cf51b0970d04ba285f29bda3c5 Mon Sep 17 00:00:00 2001 From: Cresson Remi <remi.cresson@irstea.fr> Date: Tue, 2 Jan 2024 20:40:14 +0100 Subject: [PATCH 4/7] COMP: OTB==9.0.0, TF=2.14.0 --- .gitlab-ci.yml | 2 +- RELEASE_NOTES.txt | 5 +++++ doc/docker_use.md | 19 ++++++++++++------- setup.py | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3295b83f..1ea0beeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - OTBTF_VERSION: 4.2.2 + OTBTF_VERSION: 4.3.0 OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory OTBTF_SRC: /src/otbtf # Local OTBTF source directory OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 8a63b56f..ce7b6cf8 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,3 +1,8 @@ +Version 4.3.0 (02 jan 2024) +---------------------------------------------------------------- +* Move OTB to version 9.0.0 +* Move TensorFlow to version 2.14 + Version 4.2.2 (19 oct 2023) ---------------------------------------------------------------- * Add a test to ensure that planetary computer works in the OTBTF docker image diff --git a/doc/docker_use.md b/doc/docker_use.md index b5032057..836bc134 100644 --- a/doc/docker_use.md +++ b/doc/docker_use.md @@ -25,12 +25,12 @@ Since OTBTF >= 3.2.1 you can find the latest docker images on | Name | Os | TF | OTB | Description | Dev files | Compute capability | |------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ | -| **mdl4eo/otbtf:4.2.2-cpu** | Ubuntu Jammy | r2.12 | d74ab | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.2.2-cpu-dev** | Ubuntu Jammy | r2.12 | d74ab | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.2.2-gpu** | Ubuntu Jammy | r2.12 | d74ab | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| -| **mdl4eo/otbtf:4.2.2-gpu-dev** | Ubuntu Jammy | r2.12 | d74ab | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| -| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt** | Ubuntu Jammy | r2.12 | d74ab | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| -| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt-dev** | Ubuntu Jammy | r2.12 | d74ab | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.0-cpu** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.0-cpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.0-gpu** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.3.0-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| +| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| The list of older releases is available [here](#older-images). @@ -184,4 +184,9 @@ Here you can find the list of older releases of OTBTF: | **mdl4eo/otbtf:4.2.1-gpu-dev** | Ubuntu Jammy | r2.12 | 8.1.2 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| | **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.1-gpu-opt** | Ubuntu Jammy | r2.12 | 8.1.2 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| | **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.1-gpu-opt-dev** | Ubuntu Jammy | r2.12 | 8.1.2 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| - +| **mdl4eo/otbtf:4.2.2-cpu** | Ubuntu Jammy | r2.12 | d74ab | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.2.2-cpu-dev** | Ubuntu Jammy | r2.12 | d74ab | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.2.2-gpu** | Ubuntu Jammy | r2.12 | d74ab | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf:4.2.2-gpu-dev** | Ubuntu Jammy | r2.12 | d74ab | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt** | Ubuntu Jammy | r2.12 | d74ab | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6| +| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt-dev** | Ubuntu Jammy | r2.12 | d74ab | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6| diff --git a/setup.py b/setup.py index 6c985eb0..c4d7d67d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setuptools.setup( name="otbtf", - version="4.2.2", + version="4.3.0", author="Remi Cresson", author_email="remi.cresson@inrae.fr", description="OTBTF: Orfeo ToolBox meets TensorFlow", -- GitLab From 691bab56d6429958d88a9fe4135fe2d3eb0d286f Mon Sep 17 00:00:00 2001 From: Cresson Remi <remi.cresson@irstea.fr> Date: Tue, 2 Jan 2024 20:41:59 +0100 Subject: [PATCH 5/7] DOC: update release notes for v4.3.0 --- RELEASE_NOTES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index ce7b6cf8..17df4445 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,5 +1,6 @@ Version 4.3.0 (02 jan 2024) ---------------------------------------------------------------- +* Remove OTB GUI dependencies * Move OTB to version 9.0.0 * Move TensorFlow to version 2.14 -- GitLab From 9d4207616115b2a57d516d6f71df79619883c7dc Mon Sep 17 00:00:00 2001 From: Cresson Remi <remi.cresson@irstea.fr> Date: Tue, 2 Jan 2024 21:14:00 +0100 Subject: [PATCH 6/7] COMP: update compute capability --- tools/docker/build-env-tf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh index 94546bfe..e7703f01 100644 --- a/tools/docker/build-env-tf.sh +++ b/tools/docker/build-env-tf.sh @@ -47,7 +47,7 @@ if [ ! -z $CUDA_TOOLKIT_PATH ] ; then fi export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_TOOLKIT_PATH/lib64:$CUDA_TOOLKIT_PATH/lib64/stubs" export TF_CUDA_VERSION=$(echo $CUDA_TOOLKIT_PATH | sed -r 's/.*\/cuda-(.*)/\1/') - export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.6" + export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.0,8.6,9.0" export TF_NEED_CUDA=1 export TF_CUDA_CLANG=0 export TF_NEED_TENSORRT=0 -- GitLab From e294d6de7aabae3254dd3efb8d8ec150a1f1a179 Mon Sep 17 00:00:00 2001 From: Cresson Remi <remi.cresson@irstea.fr> Date: Tue, 2 Jan 2024 21:31:20 +0100 Subject: [PATCH 7/7] DOC: update releade notes --- RELEASE_NOTES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 17df4445..28cda335 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,5 +1,6 @@ Version 4.3.0 (02 jan 2024) ---------------------------------------------------------------- +* Add compute capability 8.0 and 9.0 * Remove OTB GUI dependencies * Move OTB to version 9.0.0 * Move TensorFlow to version 2.14 -- GitLab