Add support for pyproject.toml and wsgi module paths

pip 23.1 removed the "setup.py install" fallback for projects that do
not have pyproject.toml and now uses a pyproject.toml which is vendored
in pip [1][2]. pip 24.2 has now deprecated a similar fallback to
"setup.py develop" and plans to fully remove this in pip 25.0 [3][4][5].
pbr supports editable installs since 6.0.0

pip 25.1 has now been released and the removal is complete.
by adding our own minimal pyproject.toml to ensure we are using the
correct build system.

This change also requires that we adapt how we generate our wsgi
entry point. when pyproject.toml is used the wsgi console script is
not generated in an editbale install such as is used in devstck

To adress this we need to refactor our usage of our wsgi applciation
to use a module path instead. This change does not remove
the declaration of our wsgi_scrtip entry point but it shoudl
be considered deprecated and it will be removed in the future.

To unblock the gate the devstack plugin is modifed to to deploy
using the wsgi module instead of the console script.

Finally supprot for the mod_wsgi wsgi mode is removed.
that was deprecated in devstack a few cycle ago and
support was removed in I8823e98809ed6b66c27dbcf21a00eea68ef403e8

[1] https://pip.pypa.io/en/stable/news/#v23-1
[2] https://github.com/pypa/pip/issues/8368
[3] https://pip.pypa.io/en/stable/news/#v24-2
[4] https://github.com/pypa/pip/issues/11457
[5] https://ichard26.github.io/blog/2024/08/whats-new-in-pip-24.2/
Closes-Bug: #2109608

Depends-on: https://review.opendev.org/c/openstack/watcher/+/948502
Change-Id: Iad77939ab0403c5720c549f96edfc77d2b7d90ee

devstack/lib/watcher

@@ -55,11 +55,7 @@ else
     WATCHER_BIN_DIR=$(get_python_exec_prefix)
 fi
 
-# There are 2 modes, which is "uwsgi" which runs with an apache
+WATCHER_UWSGI=watcher.wsgi.api:application
-# proxy uwsgi in front of it, or "mod_wsgi", which runs in
-# apache. mod_wsgi is deprecated, don't use it.
-WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE}
-WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi
 WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini
 
 if is_suse; then
@@ -73,11 +69,7 @@ WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322}
 WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322}
 WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
-else
-    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
-fi
 
 # Entry Points
 # ------------
@@ -101,11 +93,7 @@ function _cleanup_watcher_apache_wsgi {
 # runs that a clean run would need to clean up
 function cleanup_watcher {
     sudo rm -rf $WATCHER_STATE_PATH
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+    remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
-        remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
-    else
-        _cleanup_watcher_apache_wsgi
-    fi
 }
 
 # configure_watcher() - Set config files, create data dirs, etc
@@ -154,31 +142,6 @@ function create_watcher_accounts {
         "$WATCHER_API_URL"
 }
 
-# _config_watcher_apache_wsgi() - Set WSGI config files of watcher
-function _config_watcher_apache_wsgi {
-    local watcher_apache_conf
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-        local service_port=$WATCHER_SERVICE_PORT
-        if is_service_enabled tls-proxy; then
-            service_port=$WATCHER_SERVICE_PORT_INT
-            service_protocol="http"
-        fi
-        sudo mkdir -p $WATCHER_WSGI_DIR
-        sudo cp $WATCHER_DIR/watcher/api/app.wsgi $WATCHER_WSGI_DIR/app.wsgi
-        watcher_apache_conf=$(apache_site_config_for watcher-api)
-        sudo cp $WATCHER_DEVSTACK_FILES_DIR/apache-watcher-api.template $watcher_apache_conf
-        sudo sed -e "
-            s|%WATCHER_SERVICE_PORT%|$service_port|g;
-            s|%WATCHER_WSGI_DIR%|$WATCHER_WSGI_DIR|g;
-            s|%USER%|$STACK_USER|g;
-            s|%APIWORKERS%|$API_WORKERS|g;
-            s|%APACHE_NAME%|$APACHE_NAME|g;
-        " -i $watcher_apache_conf
-        enable_apache_site watcher-api
-   fi
-
-}
-
 # create_watcher_conf() - Create a new watcher.conf file
 function create_watcher_conf {
     # (Re)create ``watcher.conf``
@@ -196,11 +159,6 @@ function create_watcher_conf {
         iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
         iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT"
         # iniset $WATCHER_CONF api enable_ssl_api "True"
-    else
-        if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-            iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)"
-            iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
-        fi
     fi
 
     iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML
@@ -228,12 +186,8 @@ function create_watcher_conf {
     # Format logging
     setup_logging $WATCHER_CONF
 
-    #config apache files
+    write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim" "" "watcher-api"
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+
-        write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
-    else
-        _config_watcher_apache_wsgi
-    fi
     # Register SSL certificates if provided
     if is_ssl_enabled_service watcher; then
         ensure_certificates WATCHER
@@ -273,9 +227,6 @@ function install_watcherclient {
 function install_watcher {
     git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH
     setup_develop $WATCHER_DIR
-    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
-        install_apache_wsgi
-    fi
 }
 
 # start_watcher_api() - Start the API process ahead of other things
@@ -289,19 +240,10 @@ function start_watcher_api {
         service_port=$WATCHER_SERVICE_PORT_INT
         service_protocol="http"
     fi
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+    run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
-        run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF"
+    watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
-        watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
+    # TODO(sean-k-mooney): we should probably check that we can hit
-    else
+    # the microversion endpoint and get a valid response.
-        watcher_url=$service_protocol://$SERVICE_HOST:$service_port
-        enable_apache_site watcher-api
-        restart_apache_server
-        # Start proxies if enabled
-        if is_service_enabled tls-proxy; then
-            start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
-        fi
-    fi
-
     echo "Waiting for watcher-api to start..."
     if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then
         die $LINENO "watcher-api did not start"
@@ -319,12 +261,7 @@ function start_watcher {
 
 # stop_watcher() - Stop running processes (non-screen)
 function stop_watcher {
-    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+    stop_process watcher-api
-        stop_process watcher-api
-    else
-        disable_apache_site watcher-api
-        restart_apache_server
-    fi
     for serv in watcher-decision-engine watcher-applier; do
         stop_process $serv
     done

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["pbr>=6.0.0", "setuptools>=64.0.0"]
+build-backend = "pbr.build"

releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml

@@ -0,0 +1,30 @@
+---
+features:
+  - |
+    A new module, ``watcher.wsgi``, has been added as a place to gather WSGI
+    ``application`` objects. This is intended to ease deployment by providing
+    a consistent location for these objects. For example, if using uWSGI then
+    instead of:
+
+    .. code-block:: ini
+
+       [uwsgi]
+        wsgi-file = /bin/watcher-api-wsgi
+
+    You can now use:
+
+    .. code-block:: ini
+
+        [uwsgi]
+        module = watcher.wsgi.api:application
+
+    This also simplifies deployment with other WSGI servers that expect module
+    paths such as gunicorn.
+deprecations:
+  - |
+    The watcher-api-wsgi console script is deprecated for removal
+    in a future release. This artifact is generated using a setup-tools
+    extension that is provide by PBR which is also deprecated.
+    due to the changes in python packaging this custom extensions
+    is planned to be removed form all OpenStack projects in a future
+    PBR release in favor of module based wsgi applications entry points.

releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml

@@ -2,8 +2,7 @@
 fixes:
   - |
     When using prometheus datasource and more that one target has the same value
-    for the `fqdn_label`, the driver used the wrong instance label to query for host
+    for the ``fqdn_label``, the driver used the wrong instance label to query for host
-    metrics. The `instance` label is no longer used in the queries but the `fqdn_label`
+    metrics. The ``instance`` label is no longer used in the queries but the ``fqdn_label``
     which identifies all the metrics for a specific compute node.
-
+    see Bug 2103451: https://bugs.launchpad.net/watcher/+bug/2103451 for more info.
-    .. _Bug 2103451: https://bugs.launchpad.net/watcher/+bug/2103451

setup.cfg

@@ -6,6 +6,7 @@ description_file =
 author = OpenStack
 author_email = openstack-discuss@lists.openstack.org
 home_page = https://docs.openstack.org/watcher/latest/
+# TODO(sean-k-mooney): bump to >= 3.10 before m3.
 python_requires = >=3.9
 classifier =
     Environment :: OpenStack
@@ -17,7 +18,6 @@ classifier =
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3.12

tox.ini

@@ -106,8 +106,10 @@ commands =
   make -C doc/build/pdf
 
 [testenv:releasenotes]
-deps = -r{toxinidir}/doc/requirements.txt
+deps = {[testenv:docs]deps}
-commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
+commands =
+  rm -rf releasenotes/build
+  sphinx-build -W --keep-going -b html -j auto releasenotes/source releasenotes/build/html
 
 [testenv:bandit]
 skip_install = true
@@ -146,8 +148,3 @@ extension =
    N342 = checks:no_redundant_import_alias
    N366 = checks:import_stock_mock
 paths = ./watcher/hacking
-
-[doc8]
-extension=.rst
-# todo: stop ignoring doc/source/man when https://bugs.launchpad.net/doc8/+bug/1502391 is fixed
-ignore-path=doc/source/image_src,doc/source/man,doc/source/api

watcher/wsgi/api.py

@@ -0,0 +1,18 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""WSGI application entry-point for Watcher API."""
+import threading
+from watcher.api import wsgi
+application = None
+with threading.Lock():
+    if application is None:
+        application = wsgi.initialize_wsgi_app()