From a5004a14eb6bc2fd1fb21999f744c4df9b0e6483 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 4 Jun 2024 12:14:08 +0200
Subject: [PATCH 01/10] Update README.md
---
README.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index f5aa184..aaebd5e 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
# pyotb: Orfeo ToolBox for Python
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
[](https://pyotb.readthedocs.io/en/master/)
**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) in a pythonic, developer friendly
--
GitLab
From e43063aa1dc1713e2762963cb41e4e0e611efc5f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 12:20:57 +0200
Subject: [PATCH 02/10] DOC: update links to forgemia repo
---
README.md | 6 +++---
doc/index.md | 6 +++---
doc/installation.md | 2 +-
mkdocs.yml | 4 ++--
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/README.md b/README.md
index aaebd5e..93f61f5 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
# pyotb: Orfeo ToolBox for Python
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
+[](https://forgemia.inra.fr/orfeo-toolbox/pyotb/-/releases)
+[](https://forgemia.inra.fr/orfeo-toolbox/pyotb/-/commits/develop)
+[](https://forgemia.inra.fr/orfeo-toolbox/pyotb/-/commits/develop)
[](https://pyotb.readthedocs.io/en/master/)
**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) in a pythonic, developer friendly
diff --git a/doc/index.md b/doc/index.md
index 4ecfaf4..52586a4 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -37,7 +37,7 @@ on github or gitlab!
Contributions are welcome !
Open a PR/MR, or file an issue if you spot a bug or have any suggestion:
-- [Github](https://github.com/orfeotoolbox/pyotb)
-- [Orfeo ToolBox GitLab instance](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb).
+- [Github](https://github.com/orfeotoolbox/pyotb)
+- [Orfeo ToolBox GitLab instance](https://forgemia.inra.fr/orfeo-toolbox/pyotb).
-Thank you!
\ No newline at end of file
+Thank you!
diff --git a/doc/installation.md b/doc/installation.md
index 404d7bd..0414989 100644
--- a/doc/installation.md
+++ b/doc/installation.md
@@ -17,7 +17,7 @@ pip install pyotb --upgrade
For development, use the following:
```bash
-git clone https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
+git clone https://forgemia.inra.fr/orfeo-toolbox/pyotb
cd pyotb
pip install -e ".[dev]"
```
diff --git a/mkdocs.yml b/mkdocs.yml
index 88589b1..02da75c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -53,7 +53,7 @@ extra:
tabs: true
social:
- icon: fontawesome/brands/gitlab
- link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
+ link: https://forgemia.inra.fr/orfeo-toolbox/pyotb
extra_css:
- https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css
- extra.css
@@ -76,6 +76,6 @@ markdown_extensions:
# Rest of the navigation.
site_name: "pyotb: Orfeo ToolBox for Python"
-repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
+repo_url: https://forgemia.inra.fr/orfeo-toolbox/pyotb
repo_name: pyotb
docs_dir: doc/
--
GitLab
From cfbe54febfecf038d83a65584d33f8080a3519d3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Mon, 11 Mar 2024 15:38:52 +0100
Subject: [PATCH 03/10] ENH: better logging config using dict
---
pyotb/helpers.py | 43 +++++++++++++++++++++++--------------------
1 file changed, 23 insertions(+), 20 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 0e6ea2a..751016a 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -13,33 +13,36 @@ OTB_ROOT = os.environ.get("OTB_ROOT")
DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
# Logging
-# User can also get logger with `logging.getLogger("pyOTB")`
+# User can also get logger with `logging.getLogger("pyotb")`
# then use pyotb.set_logger_level() to adjust logger verbosity
-logger = logging.getLogger("pyotb")
-logger_handler = logging.StreamHandler(sys.stdout)
-formatter = logging.Formatter(
- fmt="%(asctime)s (%(levelname)-4s) [pyotb] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
-)
-logger_handler.setFormatter(formatter)
+
# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyotb level, or fallback to INFO
LOG_LEVEL = (
os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO"
)
-logger.setLevel(getattr(logging, LOG_LEVEL))
-# Here it would be possible to use a different level for a specific handler
-# A more verbose one can go to text file while print only errors to stdout
-logger_handler.setLevel(getattr(logging, LOG_LEVEL))
-logger.addHandler(logger_handler)
-
-def set_logger_level(level: str):
- """Allow user to change the current logging level.
+logger = logging.getLogger("pyotb")
- Args:
- level: logging level string ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
- """
- logger.setLevel(getattr(logging, level))
- logger_handler.setLevel(getattr(logging, level))
+logging_cfg = {
+ "version": 1,
+ "disable_existing_loggers": False,
+ "formatters": {
+ "default": {
+ "format": "%(asctime)s (%(levelname)-4s) [pyotb] %(message)s",
+ "datefmt": "%Y-%m-%d %H:%M:%S",
+ },
+ },
+ "handlers": {
+ "stdout": {
+ "class": "logging.StreamHandler",
+ "level": LOG_LEVEL,
+ "formatter": "default",
+ "stream": "ext://sys.stdout",
+ }
+ },
+ "loggers": {"pyotb": {"level": LOG_LEVEL, "handlers": ["stdout"]}},
+}
+logging.config.dictConfig(logging_cfg)
def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
--
GitLab
From 9bb289d077f32cb33d84e70ac5cbc7923ea66882 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 16:03:43 +0200
Subject: [PATCH 04/10] ENH: better logger config (preserve rootLogger state)
---
doc/managing_loggers.md | 14 +++++++-------
pyotb/__init__.py | 4 ++--
pyotb/helpers.py | 2 ++
3 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md
index 7220af4..444eb0d 100644
--- a/doc/managing_loggers.md
+++ b/doc/managing_loggers.md
@@ -4,22 +4,22 @@ Several environment variables are used in order to adjust logger level and
behaviour. It should be set before importing pyotb.
- `OTB_LOGGER_LEVEL` : used to set the default OTB logger level.
-- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set,
-- `OTB_LOGGER_LEVEL` will be used.
+- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level.
+If `PYOTB_LOGGER_LEVEL` isn't set, `OTB_LOGGER_LEVEL` will be used.
If none of those two variables is set, the logger level will be set to 'INFO'.
Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL
-You may also change the logger level after import (for pyotb only) with the
-function `set_logger_level`.
+You may also change the logger level after import (for pyotb only)
+using pyotb.logger.setLevel(level).
```python
import pyotb
-pyotb.set_logger_level('DEBUG')
+pyotb.logger.setLevel('DEBUG')
```
-Bonus : in some cases, you may want to silence the GDAL driver logger (for
-example you will see a lot of errors when reading GML files with OGR).
+Bonus : in some cases, you may want to silence the GDAL driver logger
+(for example you will see a lot of errors when reading GML files with OGR).
One useful trick is to redirect these logs to a file. This can be done using
the variable `CPL_LOG`.
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 673fd2e..7128ea6 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.2"
+__version__ = "2.0.3.dev1"
from .install import install_otb
-from .helpers import logger, set_logger_level
+from .helpers import logger
from .core import (
OTBObject,
App,
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 751016a..9829615 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,5 +1,7 @@
"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install."""
+
import logging
+import logging.config
import os
import sys
import sysconfig
--
GitLab
From ef4092d27e076b5ec267604ac90738ff8dc8ec27 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 16:12:23 +0200
Subject: [PATCH 05/10] CI: pylint ignore functions.py
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 715e8a3..c7015fb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,7 +45,7 @@ pylint:
before_script:
- python3 -m pip install pylint
script:
- - pylint $PWD/pyotb --disable=fixme
+ - pylint $PWD/pyotb --disable=fixme --ignore=functions.py
codespell:
extends: .static_analysis
--
GitLab
From d2388a3bc800bd743d2f2f410353529d55f6a44d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 16:30:18 +0200
Subject: [PATCH 06/10] ENH: logger levels
---
doc/managing_loggers.md | 18 ++++++++++++++++++
pyotb/helpers.py | 2 +-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md
index 444eb0d..20a6725 100644
--- a/doc/managing_loggers.md
+++ b/doc/managing_loggers.md
@@ -23,6 +23,24 @@ Bonus : in some cases, you may want to silence the GDAL driver logger
One useful trick is to redirect these logs to a file. This can be done using
the variable `CPL_LOG`.
+## Log to file
+It is possible to change the behaviour of the default pyotb logger as follow
+
+```py
+import logging
+import pyotb
+# Optional : remove default stdout handler (but OTB will still print its own log)
+pyotb.logger.handlers.pop()
+# Add file handler
+handler = logging.FileHandler("/my/log/file.log")
+handler.setLevel("DEBUG")
+pyotb.logger.addHandler(handler)
+```
+
+For more advanced configuration and to manage conflicts between several loggers,
+see the [logging module docs](https://docs.python.org/3/howto/logging-cookbook.html)
+and use the `dictConfig()` function to configure your own logger.
+
## Named applications in logs
It is possible to change an app name in order to track it easily in the logs :
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 9829615..9653615 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -37,7 +37,7 @@ logging_cfg = {
"handlers": {
"stdout": {
"class": "logging.StreamHandler",
- "level": LOG_LEVEL,
+ "level": "DEBUG",
"formatter": "default",
"stream": "ext://sys.stdout",
}
--
GitLab
From de801eae7e2bd80706801df4a48b23998136a5cd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 16:52:59 +0200
Subject: [PATCH 07/10] CI: bump version
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 7128ea6..594272b 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.3.dev1"
+__version__ = "2.0.3.dev2"
from .install import install_otb
from .helpers import logger
--
GitLab
From d9dacde7cd79c68174d0445be196382119ab1acc Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 17:03:46 +0200
Subject: [PATCH 08/10] CI: add coverage config
---
.coveragerc | 6 ++++++
.gitlab-ci.yml | 1 +
2 files changed, 7 insertions(+)
create mode 100644 .coveragerc
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..927313b
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,6 @@
+[run]
+omit =
+ pyotb/depreciation.py
+ pyotb/functions.py
+ pyotb/helpers.py
+ pyotb/install.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7015fb..c932c9c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -74,6 +74,7 @@ test_install:
- changes:
- "**/*.py"
- .gitlab-ci.yml
+ - .coveragerc
variables:
SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif
PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif
--
GitLab
From bc73e9c7464c281c1fc7a72097cb5c680780eb54 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@gmail.com>
Date: Tue, 4 Jun 2024 17:23:10 +0200
Subject: [PATCH 09/10] CI: use forgemia stable runners tag
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c932c9c..b91c24c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,5 @@
default:
+ tags: [stable]
image: mdl4eo/otbtf:4.3.0-cpu
interruptible: true
--
GitLab
From f1c5ac8a9310c2db7c18ccc13002ec1f1c0293c8 Mon Sep 17 00:00:00 2001
From: Cresson Remi <remi.cresson@irstea.fr>
Date: Wed, 9 Oct 2024 19:49:49 +0200
Subject: [PATCH 10/10] Resolve "Memory leak"
---
RELEASE_NOTES.txt | 8 ++++++--
pyotb/__init__.py | 2 +-
pyotb/core.py | 28 ++++++++++++----------------
3 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 7a36b43..ff64cbe 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,9 @@
+---------------------------------------------------------------------
+2.1.0 (Oct 9, 2024) - Changes since version 2.0.2
+
+- Fix memory leak due to circular references to Output objects in list App.outputs
+- Breaking change : replaced App.outputs by a tuple of out image keys (App._out_image_keys)
+
---------------------------------------------------------------------
2.0.2 (Apr 5, 2024) - Changes since version 2.0.1
@@ -5,13 +11,11 @@
- Fix a bug with parameters of type "field" for vector files
- Fix wrong output parameter key in ImageClassifier and ImageClassifierFromDeepFeatures
-
---------------------------------------------------------------------
2.0.1 (Dec 18, 2023) - Changes since version 2.0.0
- Fix a bug when writing outputs in uint8
-
---------------------------------------------------------------------
2.0.0 (Nov 23, 2023) - Changes since version 1.5.4
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 594272b..5e58316 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.3.dev2"
+__version__ = "2.1.0"
from .install import install_otb
from .helpers import logger
diff --git a/pyotb/core.py b/pyotb/core.py
index b78efd4..ee73fcb 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -579,7 +579,7 @@ class App(OTBObject):
self._exports_dic = {}
self._settings, self._auto_parameters = {}, {}
self._time_start, self._time_end = 0.0, 0.0
- self.data, self.outputs = {}, {}
+ self.data = {}
self.quiet, self.frozen = quiet, frozen
# Param keys and types
@@ -597,17 +597,15 @@ class App(OTBObject):
for key in self.parameters_keys
if self.app.GetParameterType(key) == otb.ParameterType_Choice
}
+ self._out_image_keys = tuple(
+ key
+ for key, param in self._out_param_types.items()
+ if param == otb.ParameterType_OutputImage
+ )
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
self.set_parameters(*args, **kwargs)
- # Create Output image objects
- for key in (
- key
- for key, param in self._out_param_types.items()
- if param == otb.ParameterType_OutputImage
- ):
- self.outputs[key] = Output(self, key, self._settings.get(key))
if not self.frozen:
self.execute()
@@ -643,8 +641,8 @@ class App(OTBObject):
return self._all_param_types[key] in param_types
def __is_multi_output(self):
- """Check if app has multiple outputs to ensure re-execution during write()."""
- return len(self.outputs) > 1
+ """Check if app has multiple image outputs to ensure re-execution in write()."""
+ return len(self._out_image_keys) > 1
def is_input(self, key: str) -> bool:
"""Returns True if the parameter key is an input."""
@@ -745,10 +743,8 @@ class App(OTBObject):
f"{self.name}: error before execution,"
f" while setting '{key}' to '{obj}': {e})"
) from e
- # Save / update setting value and update the Output object initialized in __init__ without a filepath
+ # Save / update setting value
self._settings[key] = obj
- if key in self.outputs:
- self.outputs[key].filepath = obj
if key in self._auto_parameters:
del self._auto_parameters[key]
@@ -1104,8 +1100,8 @@ class App(OTBObject):
if isinstance(key, str):
if key in self.data:
return self.data[key]
- if key in self.outputs:
- return self.outputs[key]
+ if key in self._out_image_keys:
+ return Output(self, key, self._settings.get(key))
if key in self.parameters:
return self.parameters[key]
raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'")
@@ -1538,7 +1534,7 @@ class Output(OTBObject):
mkdir: bool = True,
):
"""Constructor for an Output object, initialized during App.__init__."""
- self.parent_pyotb_app = pyotb_app # keep trace of parent app
+ self.parent_pyotb_app = pyotb_app # keep a reference to parent app
self.param_key = param_key
self.filepath = filepath
if mkdir and filepath is not None:
--
GitLab