diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c89242c37cc320dd0b1828e4b65d86fb60db20a..5cfe0b9eef546c2b4f0c7be595f7c15d394dbae6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: gitlab-registry.irstea.fr/remi.cresson/otbtf/otbtf3.0:cpu-basic-testing +image: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.1-cpu-basic-testing variables: OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory diff --git a/Dockerfile b/Dockerfile index 34b9b4a4adba7587f73e1d3daae8511cbb3c69b8..28ee7875d9c80324d48f3b476371d89c1b53bb25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ RUN if $GUI; then \ ### Python3 links and pip packages RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip # NumPy version is conflicting with system's gdal dep and may require venv -ARG NUMPY_SPEC="==1.19.*" +ARG NUMPY_SPEC="==1.22.*" RUN pip install --no-cache-dir -U pip wheel mock six future deprecated "numpy$NUMPY_SPEC" \ && pip install --no-cache-dir --no-deps keras_applications keras_preprocessing @@ -40,7 +40,7 @@ WORKDIR /src/tf RUN git config --global advice.detachedHead false ### TF -ARG TF=v2.5.0 +ARG TF=v2.8.0 # Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default) RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \ && chmod +x /opt/otbtf/bin/bazelisk \ diff --git a/README.md b/README.md index 1a462fe58d1283e6cde8fbd9997998601de2bb44..d1993d5bb839fea504524e9cd21c4d120bf07335 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,7 @@ For now you have two options: either use the existing **docker image**, or build Use the latest image from dockerhub: ``` -docker pull mdl4eo/otbtf3.0:cpu -docker run -u otbuser -v $(pwd):/home/otbuser mdl4eo/otbtf3.0:cpu otbcli_PatchesExtraction -help +docker run mdl4eo/otbtf3.1:cpu-basic otbcli_PatchesExtraction -help ``` Read more in the [docker use documentation](doc/DOCKERUSE.md). diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 764033051413016bf2de6441b15a915fe76840a0..f9934e43937077b67096608f5aba02c1b1e6e1ee 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,3 +1,10 @@ +Version 3.1 (5 apr 2022) +---------------------------------------------------------------- +* Add the "split" strategy in the `PatchesSelection` application. Now the application can select points for 3 independent data splits (train, valid, test). +* `PatchesExtraction` and `PatchesSelection` don't use the `usenodata` option anymore. Now to use nodata, one simply has to set a parameter value for the nodata (no more default value). +* New python examples in `python/examples/tensorflow_v2x` to illustrate how deterministic models can be quickly built using TensorFlow 2, exporte as SavedModel, and used with OTBTF. +* Change "import gdal" --> "from osgeo import gdal" in otbtf.py + Version 3.0.0-beta (20 nov 2021) ---------------------------------------------------------------- * Use Tensorflow 2 API everywhere. Everything is backward compatible (old models can still be used). diff --git a/app/otbPatchesExtraction.cxx b/app/otbPatchesExtraction.cxx index b00c3a1706ae32973c3f98e96b3acb5b4621939c..33eb603ce9ccef5231e80845b93159611ebdb5c7 100644 --- a/app/otbPatchesExtraction.cxx +++ b/app/otbPatchesExtraction.cxx @@ -91,7 +91,7 @@ public: ss_key_dims_y << ss_group_key.str() << ".patchsizey"; ss_desc_dims_y << "Y patch size for image " << inputNumber; ss_key_nodata << ss_group_key.str() << ".nodata"; - ss_desc_nodata << "No-data value for image " << inputNumber << "(used only if \"usenodata\" is on)"; + ss_desc_nodata << "No-data value for image " << inputNumber; // Populate group AddParameter(ParameterType_Group, ss_group_key.str(), ss_desc_group.str()); diff --git a/doc/APPLICATIONS.md b/doc/APPLICATIONS.md index a0bd988f2a39da241292cbc1be9e918c3fa6f23a..2ca5fd1f7485108ad34fa201f10779dfbbc4d5e5 100644 --- a/doc/APPLICATIONS.md +++ b/doc/APPLICATIONS.md @@ -30,9 +30,8 @@ MISSING -source1.il <string list> Input image(s) 1 (mandatory) MISSING -source1.out <string> [pixel] Output patches for image 1 [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is float) (mandatory) MISSING -source1.patchsizex <int32> X patch size for image 1 (mandatory) MISSING -source1.patchsizey <int32> Y patch size for image 1 (mandatory) - -source1.nodata <float> No-data value for image 1(used only if "usenodata" is on) (mandatory, default value is 0) + -source1.nodata <float> No-data value for image 1 MISSING -vec <string> Positions of the samples (must be in the same projection as input image) (mandatory) - -usenodata <boolean> Reject samples that have no-data value (optional, off by default, default value is false) -outlabels <string> [pixel] output labels [pixel=uint8/uint16/int16/uint32/int32/float/double/cint16/cint32/cfloat/cdouble] (default value is uint8) (optional, off by default) MISSING -field <string> field of class in the vector data (mandatory) -progress <boolean> Report progress diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md index 34e58cc3eab4a170e8e4abd445da8a826890b8bb..69382c9613dd988f0a75d4c1b8eff0cdb487529c 100644 --- a/doc/DOCKERUSE.md +++ b/doc/DOCKERUSE.md @@ -24,8 +24,14 @@ Here is the list of OTBTF docker images hosted on [dockerhub](https://hub.docker | **mdl4eo/otbtf3.0:cpu-basic-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| | **mdl4eo/otbtf3.0:gpu** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6| | **mdl4eo/otbtf3.0:gpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6| - -- `cpu` tagged docker images are compiled without optimization. +| **mdl4eo/otbtf3.1:cpu-basic** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf3.1:cpu-basic-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf3.1:gpu-basic** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf3.1:gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | yes | 5.2,6.1,7.0,7.5,8.6| +| **mdl4eo/otbtf3.1:gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6| + +- `cpu` tagged docker images are compiled for CPU usage only. +- `xxx-basic` tagged docker images are compiled without CPU optimization flags. Other images are compiled with AVX, SSE, FMA. - `gpu` tagged docker images are suited for **NVIDIA GPUs**. They use CUDA/CUDNN support. - `cpu-mkl` tagged docker image is experimental, it is optimized for Intel CPUs with AVX512 flags. @@ -49,7 +55,7 @@ For instance, suppose you have some data in `/mnt/my_device/` that you want to u The following command shows you how to access the folder from the docker image. ```bash -docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf2.5:cpu bash -c "ls /data" +docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf3.1:cpu-basic bash -c "ls /data" ``` Beware of ownership issues! see the last section of this doc. @@ -62,13 +68,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime* With Docker version earlier than 19.03 : ```bash -docker run --runtime=nvidia -ti mdl4eo/otbtf2.5:gpu bash +docker run --runtime=nvidia -ti mdl4eo/otbtf3.1:gpu bash ``` With Docker version including and after 19.03 : ```bash -docker run --gpus all -ti mdl4eo/otbtf2.5:gpu bash +docker run --gpus all -ti mdl4eo/otbtf3.1:gpu bash ``` You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/). @@ -81,7 +87,7 @@ Be careful though, these infos might be a bit outdated... 1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux) 2. Install [docker desktop](https://www.docker.com/products/docker-desktop) 3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine* -3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf2.5:cpu` +3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf3.1:cpu` 4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu  5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up! @@ -130,12 +136,12 @@ sudo systemctl {status,enable,disable,start,stop} docker Run a simple command in a one-shot container: ```bash -docker run mdl4eo/otbtf2.5:cpu otbcli_PatchesExtraction +docker run mdl4eo/otbtf3.1:cpu otbcli_PatchesExtraction ``` You can also use the image in interactive mode with bash: ```bash -docker run -ti mdl4eo/otbtf2.5:cpu bash +docker run -ti mdl4eo/otbtf3.1:cpu bash ``` ### Persistent container @@ -145,7 +151,7 @@ Beware of ownership issues, see the last section of this doc. ```bash docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \ - --name otbtf mdl4eo/otbtf2.5:cpu /bin/bash + --name otbtf mdl4eo/otbtf3.1:cpu /bin/bash ``` ### Interactive session @@ -209,7 +215,7 @@ Create a named container (here with your HOME as volume), Docker will automatica ```bash docker create --interactive --tty --volume /home/$USER:/home/otbuser \ - --name otbtf mdl4eo/otbtf2.5:cpu /bin/bash + --name otbtf mdl4eo/otbtf3.1:cpu /bin/bash ``` Start a background container process: