Add no oauth middleware to bypass keystone authentication

This PS adds noauth middleware to bypass keystone authentication
which will occur when Deckhand's server is executed in development
mode. Development mode is enabled by setting development_mode as True
in etc/deckhand/deckhand.conf.sample.

The logic is similar to Drydock's here: [0].

[0] 1c78477e95/drydock_provisioner/util.py (L43)

Co-Authored-By: Luna Das <luna.das@imaginea.com>
Co-Authored-By: Felipe Monteiro <felipe.monteiro@att.com>
Change-Id: I677d3d92768e0aa1a550772700403e0f028b0c59

deckhand/conf/config.py

@@ -36,22 +36,12 @@ barbican_opts = [
 
 
 default_opts = [
-    cfg.BoolOpt('allow_anonymous_access', default=False,
-                help="""
-Allow limited access to unauthenticated users.
-
-Assign a boolean to determine API access for unauthenticated
-users. When set to False, the API cannot be accessed by
-unauthenticated users. When set to True, unauthenticated users can
-access the API with read-only privileges.
-
-Possible values:
-    * True
-    * False
-"""),
     cfg.BoolOpt('profiler', default=False,
-                help="Enabling profiling of API requests. Do NOT "
-                     "use in production."),
+                help="Enables profiling of API requests. Do NOT use in "
+                     "production."),
+    cfg.BoolOpt('development_mode', default=False,
+                help="Enables development mode, which disables Keystone "
+                     "authentication. Do NOT use in production.")
 ]
 
 

deckhand/control/api.py

@@ -27,22 +27,43 @@ CONF = cfg.CONF
 logging.register_options(CONF)
 LOG = logging.getLogger(__name__)
 
-CONFIG_FILES = ['deckhand.conf', 'deckhand-paste.ini']
+CONFIG_FILES = {
+    'conf': 'deckhand.conf',
+    'paste': 'deckhand-paste.ini'
+}
+_NO_AUTH_CONFIG = 'noauth-paste.ini'
 
 
 def _get_config_files(env=None):
     if env is None:
         env = os.environ
+
+    config_files = CONFIG_FILES.copy()
     dirname = env.get('DECKHAND_CONFIG_DIR', '/etc/deckhand').strip()
-    return [os.path.join(dirname, config_file) for config_file in CONFIG_FILES]
+
+    # Workaround the fact that this reads from a config file to determine which
+    # paste.ini file to use for server instantiation. This chicken and egg
+    # problem is solved by using ConfigParser below.
+    conf_path = os.path.join(dirname, config_files['conf'])
+    temp_conf = {}
+    config_parser = cfg.ConfigParser(conf_path, temp_conf)
+    config_parser.parse()
+    use_development_mode = (
+        temp_conf['DEFAULT'].get('development_mode') == ['true']
+    )
+
+    if use_development_mode:
+        config_files['paste'] = _NO_AUTH_CONFIG
+        LOG.warning('Development mode enabled - Keystone authentication '
+                    'disabled.')
+
+    return {
+        key: os.path.join(dirname, file) for key, file in config_files.items()
+    }
 
 
 def setup_logging(conf):
-    # Add additional dependent libraries that have unhelp bug levels
-    extra_log_level_defaults = []
-
-    logging.set_defaults(default_log_levels=logging.get_default_log_levels() +
-                         extra_log_level_defaults)
+    logging.set_defaults(default_log_levels=logging.get_default_log_levels())
     logging.setup(conf, 'deckhand')
     py_logging.captureWarnings(True)
 
@@ -53,9 +74,12 @@ def init_application():
     Create routes for the v1.0 API and sets up logging.
     """
     config_files = _get_config_files()
-    paste_file = config_files[-1]
+    paste_file = config_files['paste']
+
+    CONF([],
+         project='deckhand',
+         default_config_files=list(config_files.values()))
 
-    CONF([], project='deckhand', default_config_files=config_files)
     setup_logging(CONF)
 
     policy.Enforcer(CONF)

deckhand/control/middleware.py

@@ -54,7 +54,7 @@ class ContextMiddleware(object):
             if req.headers.get('X-IDENTITY-STATUS') == 'Confirmed':
                 req.context = deckhand.context.RequestContext.from_environ(
                     req.env)
-            elif CONF.allow_anonymous_access:
+            elif CONF.development_mode:
                 req.context = deckhand.context.get_context()
             else:
                 raise falcon.HTTPUnauthorized()

deckhand/control/no_oauth_middleware.py

@@ -0,0 +1,63 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class NoAuthFilter(object):
+    """PasteDeploy filter for NoAuth to be used in testing."""
+
+    def __init__(self, app, forged_roles=None):
+        self.app = app
+        self.forged_roles = forged_roles or ('admin',)
+
+    def __call__(self, environ, start_response):
+        """Forge headers to make unauthenticated requests look authenticated.
+
+        If the request has a X-AUTH-TOKEN header,
+        assume it is a valid request and
+        noop. Otherwise forge Keystone middleware headers so the
+        request looks valid
+        with the configured forged roles.
+        """
+        if 'HTTP_X_AUTH_TOKEN' in environ:
+            return self.app(environ, start_response)
+
+        environ['HTTP_X_IDENTITY_STATUS'] = 'Confirmed'
+
+        for envvar in [
+                'USER_NAME', 'USER_ID', 'USER_DOMAIN_ID', 'PROJECT_ID',
+                'PROJECT_DOMAIN_NAME'
+        ]:
+            varname = "HTTP_X_%s" % envvar
+            environ[varname] = 'noauth'
+
+        if 'admin' in self.forged_roles:
+            environ['HTTP_X_IS_ADMIN_PROJECT'] = 'True'
+        else:
+            environ['HTTP_X_IS_ADMIN_PROJECT'] = 'False'
+        environ['HTTP_X_ROLES'] = ','.join(self.forged_roles)
+
+        return self.app(environ, start_response)
+
+
+def noauth_filter_factory(global_conf, forged_roles):
+    """Create a NoAuth paste deploy filter
+
+    :param forged_roles: A space seperated list for roles to forge on requests
+    """
+    forged_roles = forged_roles.split()
+
+    def filter(app):
+        return NoAuthFilter(app, forged_roles)
+
+    return filter

deckhand/tests/deckhand.conf.test

@@ -2,10 +2,7 @@
 debug = true
 publish_errors = true
 use_stderr = true
-# NOTE: allow_anonymous_access allows these functional tests to get around
-# Keystone authentication, but the context that is provided has zero privileges
-# so we must also override the policy file for authorization to pass.
-allow_anonymous_access = true
+development_mode = false
 
 [oslo_policy]
 policy_file = policy.yaml

deckhand/tests/unit/base.py

@@ -38,6 +38,8 @@ class DeckhandTestCase(testtools.TestCase):
         self.useFixture(fixtures.FakeLogger('deckhand'))
         self.useFixture(dh_fixtures.ConfPatcher(
             api_endpoint='http://127.0.0.1/key-manager', group='barbican'))
+        self.useFixture(dh_fixtures.ConfPatcher(
+            development_mode=True, group=None))
 
     def override_config(self, name, override, group=None):
         CONF.set_override(name, override, group)

deckhand/tests/unit/control/base.py

@@ -31,9 +31,9 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase,
         self.app = falcon_testing.TestClient(
             service.deckhand_app_factory(None))
         self.policy = self.useFixture(fixtures.RealPolicyFixture())
-        # NOTE: allow_anonymous_access allows these unit tests to get around
+        # NOTE: development_mode allows these unit tests to get around
         # Keystone authentication.
-        self.useFixture(fixtures.ConfPatcher(allow_anonymous_access=True))
+        self.useFixture(fixtures.ConfPatcher(development_mode=True))
 
     def _read_data(self, file_name):
         # Reads data from a file in the resources directory

deckhand/tests/unit/control/test_api_initialization.py

@@ -49,10 +49,12 @@ class TestApi(test_base.DeckhandTestCase):
         curr_path = os.path.dirname(os.path.realpath(__file__))
         repo_path = os.path.join(
             curr_path, os.pardir, os.pardir, os.pardir, os.pardir)
-        temp_config_files = [
-            os.path.join(repo_path, 'etc', 'deckhand', 'deckhand.conf.sample'),
-            os.path.join(repo_path, 'etc', 'deckhand', 'deckhand-paste.ini')
-        ]
+        temp_config_files = {
+            'conf': os.path.join(
+                repo_path, 'etc', 'deckhand', 'deckhand.conf.sample'),
+            'paste': os.path.join(
+                repo_path, 'etc', 'deckhand', 'deckhand-paste.ini')
+        }
         mock_get_config_files = self.patchobject(
             api, '_get_config_files', autospec=True)
         mock_get_config_files.return_value = temp_config_files

docs/source/getting-started.rst

@@ -175,12 +175,37 @@ Substitute the connection information into the config file in
 Finally, run Deckhand::
 
     $ chmod +x entrypoint.sh
-    $ ./entrypoint.sh
+    $ ./entrypoint.sh server
 
 To kill the ephemeral DB afterward::
 
     $ pifpaf_stop
 
+Development Mode
+----------------
+
+Development mode means running Deckhand without Keystone authentication.
+Note that enabling development mode will effectively disable all authN
+and authZ in Deckhand.
+
+To enable development mode, add the following to the ``deckhand.conf``
+inside ``$CONF_DIR``:
+
+.. code-block:: ini
+
+  [DEFAULT]
+  development_mode = True
+
+After, from the command line, execute:
+
+.. code-block:: console
+
+    $ [sudo] docker run --rm \
+        --net=host \
+        -p 9000:9000 \
+        -v $CONF_DIR:/etc/deckhand \
+        quay.io/attcomdev/deckhand:latest server
+
 Development Utilities
 ---------------------
 

etc/deckhand/deckhand.conf.sample

@@ -4,20 +4,12 @@
 # From deckhand.conf
 #
 
-#
-# Allow limited access to unauthenticated users.
-#
-# Assign a boolean to determine API access for unathenticated
-# users. When set to False, the API cannot be accessed by
-# unauthenticated users. When set to True, unauthenticated users can
-# access the API with read-only privileges. This however only applies
-# when using ContextMiddleware.
-#
-# Possible values:
-#     * True
-#     * False
-#  (boolean value)
-#allow_anonymous_access = false
+# Enables profiling of API requests. Do NOT use in production. (boolean value)
+#profiler = false
+
+# Enables development mode, which disables Keystone authentication. Do NOT use
+# in production. (boolean value)
+#development_mode = false
 
 #
 # From oslo.log
@@ -76,6 +68,10 @@
 # log_config_append is set. (string value)
 #syslog_log_facility = LOG_USER
 
+# Use JSON formatting for logging. This option is ignored if log_config_append
+# is set. (boolean value)
+#use_json = false
+
 # Log output to standard error. This option is ignored if log_config_append is
 # set. (boolean value)
 #use_stderr = false
@@ -165,6 +161,9 @@
 # Authentication URL (string value)
 #auth_url = <None>
 
+# Scope for system operations (string value)
+#system_scope = <None>
+
 # Domain ID to scope to (string value)
 #domain_id = <None>
 
@@ -337,6 +336,10 @@
 # raised. Set to -1 to specify an infinite retry count. (integer value)
 #db_max_retries = 20
 
+# Optional URL parameters to append onto the connection URL at connect time;
+# specify as param1=value1&param2=value2&... (string value)
+#connection_parameters =
+
 
 [healthcheck]
 
@@ -379,6 +382,22 @@
 # you're using a versioned v2 endpoint here, then this should *not* be the same
 # endpoint the service user utilizes for validating tokens, because normal end
 # users may not be able to reach that endpoint. (string value)
+# Deprecated group/name - [keystone_authtoken]/auth_uri
+#www_authenticate_uri = <None>
+
+# DEPRECATED: Complete "public" Identity API endpoint. This endpoint should not
+# be an "admin" endpoint, as it should be accessible by all end users.
+# Unauthenticated clients are redirected to this endpoint to authenticate.
+# Although this endpoint should ideally be unversioned, client support in the
+# wild varies. If you're using a versioned v2 endpoint here, then this should
+# *not* be the same endpoint the service user utilizes for validating tokens,
+# because normal end users may not be able to reach that endpoint. This option
+# is deprecated in favor of www_authenticate_uri and will be removed in the S
+# release. (string value)
+# This option is deprecated for removal since Queens.
+# Its value may be silently ignored in the future.
+# Reason: The auth_uri option is deprecated in favor of www_authenticate_uri and
+# will be removed in the S  release.
 #auth_uri = <None>
 
 # API version of the admin Identity API endpoint. (string value)
@@ -451,7 +470,10 @@
 # in the cache. If ENCRYPT, token data is encrypted and authenticated in the
 # cache. If the value is not one of these options or empty, auth_token will
 # raise an exception on initialization. (string value)
-# Allowed values: None, MAC, ENCRYPT
+# Possible values:
+# None - <No description provided>
+# MAC - <No description provided>
+# ENCRYPT - <No description provided>
 #memcache_security_strategy = None
 
 # (Optional, mandatory if memcache_security_strategy is defined) This string is
@@ -568,6 +590,14 @@
 # From oslo.policy
 #
 
+# This option controls whether or not to enforce scope when evaluating policies.
+# If ``True``, the scope of the token used in the request is compared to the
+# ``scope_types`` of the policy being enforced. If the scopes do not match, an
+# ``InvalidScope`` exception will be raised. If ``False``, a message will be
+# logged informing operators that policies are being invoked with mismatching
+# scope. (boolean value)
+#enforce_scope = false
+
 # The file that defines policies. (string value)
 #policy_file = policy.json
 
@@ -580,3 +610,22 @@
 # directories to be searched.  Missing or empty directories are ignored. (multi
 # valued)
 #policy_dirs = policy.d
+
+# Content Type to send and receive data for REST based policy check (string
+# value)
+# Possible values:
+# application/x-www-form-urlencoded - <No description provided>
+# application/json - <No description provided>
+#remote_content_type = application/x-www-form-urlencoded
+
+# server identity verification for REST based policy check (boolean value)
+#remote_ssl_verify_server_crt = false
+
+# Absolute path to ca cert file for REST based policy check (string value)
+#remote_ssl_ca_crt_file = <None>
+
+# Absolute path to client cert for REST based policy check (string value)
+#remote_ssl_client_crt_file = <None>
+
+# Absolute path client key file REST based policy check (string value)
+#remote_ssl_client_key_file = <None>

etc/deckhand/noauth-paste.ini

@@ -0,0 +1,35 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# PasteDeploy configuration file without Keystone authentication.
+
+[app:api]
+paste.app_factory = deckhand.service:deckhand_app_factory
+
+[filter:noauth]
+forged_roles = admin
+paste.filter_factory = deckhand.control.no_oauth_middleware:noauth_filter_factory
+
+[filter:debug]
+use = egg:oslo.middleware#debug
+
+[filter:cors]
+paste.filter_factory = oslo_middleware.cors:filter_factory
+oslo_config_project = deckhand
+
+[filter:request_id]
+paste.filter_factory = oslo_middleware:RequestId.factory
+
+[pipeline:deckhand_api]
+pipeline = noauth api

releasenotes/notes/development-mode-51208c39e9eee34f.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Development mode has been added to Deckhand, allowing for the possibility
+    of running Deckhand without Keystone. A new paste file has been added
+    to ``etc/deckhand`` called ``noauth-paste.ini`` which excludes Keystone
+    authentication. To run Deckhand in development mode, set development_mode
+    as True in deckhand.conf. Note that Deckhand will expect to find
+    ``noauth-paste.ini`` on the host with development_mode set as True in
+    etc/deckhand/deckhand.conf.sample.

tools/common-tests.sh

@@ -10,6 +10,16 @@ function log_section {
 
 
 function deploy_postgre {
+    #######################################
+    # Deploy an ephemeral PostgreSQL DB.
+    # Globals:
+    #   POSTGRES_ID
+    #   POSTGRES_IP
+    # Arguments:
+    #   None
+    # Returns:
+    #   None
+    #######################################
     set -xe
 
     POSTGRES_ID=$(
@@ -31,6 +41,19 @@ function deploy_postgre {
 
 
 function gen_config {
+    #######################################
+    # Generate sample configuration file
+    # Globals:
+    #   CONF_DIR
+    #   DECKHAND_TEST_URL
+    #   DATABASE_URL
+    #   DECKHAND_CONFIG_DIR
+    # Arguments:
+    #   disable_keystone: true or false
+    #   Deckhand test URL: URL to Deckhand wsgi server
+    # Returns:
+    #   None
+    #######################################
     set -xe
 
     log_section "Creating config directory and test deckhand.conf"
@@ -38,7 +61,8 @@ function gen_config {
     CONF_DIR=$(mktemp -d -p $(pwd))
     sudo chmod 777 -R $CONF_DIR
 
-    export DECKHAND_TEST_URL=$1
+    local disable_keystone=$1
+    export DECKHAND_TEST_URL=$2
     export DATABASE_URL=postgresql+psycopg2://deckhand:password@$POSTGRES_IP:5432/deckhand
     # Used by Deckhand's initialization script to search for config files.
     export DECKHAND_CONFIG_DIR=$CONF_DIR
@@ -54,6 +78,11 @@ function gen_config {
         sed '1 a log_config_append = '"$CONF_DIR"'/logging.conf' $conf_file
     fi
 
+    if $disable_keystone; then
+        log_section "Toggling development_mode on to disable Keystone authentication."
+        sed -i -e 's/development_mode = false/development_mode = true/g' $conf_file
+    fi
+
     echo $conf_file 1>&2
     cat $conf_file 1>&2
 
@@ -63,33 +92,23 @@ function gen_config {
 
 
 function gen_paste {
+    #######################################
+    # Generate sample paste.ini file
+    # Globals:
+    #   CONF_DIR
+    # Arguments:
+    #   disable_keystone: true or false
+    # Returns:
+    #   None
+    #######################################
     set -xe
 
     local disable_keystone=$1
 
     if $disable_keystone; then
-        log_section Disabling Keystone authentication.
-        sed 's/authtoken api/api/' etc/deckhand/deckhand-paste.ini &> $CONF_DIR/deckhand-paste.ini
+        log_section "Using noauth-paste.ini to disable Keystone authentication."
+        cp etc/deckhand/noauth-paste.ini $CONF_DIR/noauth-paste.ini
     else
         cp etc/deckhand/deckhand-paste.ini $CONF_DIR/deckhand-paste.ini
     fi
 }
-
-
-function gen_policy {
-    set -xe
-
-    log_section "Creating policy file with liberal permissions"
-
-    policy_file='etc/deckhand/policy.yaml.sample'
-    policy_pattern="deckhand\:"
-
-    touch $CONF_DIR/policy.yaml
-
-    sed -n "/$policy_pattern/p" "$policy_file" \
-        | sed 's/^../\"/' \
-        | sed 's/rule\:[A-Za-z\_\-]*/@/' > $CONF_DIR/policy.yaml
-
-    echo $CONF_DIR/'policy.yaml' 1>&2
-    cat $CONF_DIR/'policy.yaml' 1>&2
-}

tools/functional-tests.sh

@@ -46,9 +46,8 @@ trap cleanup_deckhand EXIT
 
 
 function deploy_deckhand {
-    gen_config "http://localhost:9000"
+    gen_config true "127.0.0.1:9000"
     gen_paste true
-    gen_policy
 
     if [ -z "$DECKHAND_IMAGE" ]; then
         log_section "Running Deckhand via uwsgi."
@@ -64,6 +63,13 @@ function deploy_deckhand {
         source $ROOTDIR/../entrypoint.sh server &
     else
         log_section "Running Deckhand via Docker."
+
+        # If container is already running, kill it.
+        DECKHAND_ID=$(sudo docker ps --filter ancestor=$DECKHAND_IMAGE --format "{{.ID}}")
+        if [ -n "$DECKHAND_ID" ]; then
+            sudo docker stop $DECKHAND_ID
+        fi
+
         sudo docker run \
             --rm \
             --net=host \
@@ -75,13 +81,13 @@ function deploy_deckhand {
             -p 9000:9000 \
             -v $CONF_DIR:/etc/deckhand \
             $DECKHAND_IMAGE server &> $STDOUT &
+
+        DECKHAND_ID=$(sudo docker ps | grep deckhand | awk '{print $1}')
+        echo $DECKHAND_ID
     fi
 
     # Give the server a chance to come up. Better to poll a health check.
     sleep 5
-
-    DECKHAND_ID=$(sudo docker ps | grep deckhand | awk '{print $1}')
-    echo $DECKHAND_ID
 }
 
 

tools/integration-tests.sh

@@ -141,7 +141,7 @@ function deploy_deckhand {
     openstack service list | grep deckhand
     openstack endpoint list | grep deckhand
 
-    gen_config $deckhand_endpoint
+    gen_config false $deckhand_endpoint
     gen_paste false
 
     # NOTE(fmontei): Generate an admin token instead of hacking a policy

tox.ini

@@ -20,20 +20,20 @@ commands =
 
 [testenv:py27]
 commands =
-    {[testenv]commands}
-    stestr run {posargs}
-    stestr slowest
+  {[testenv]commands}
+  stestr run {posargs}
+  stestr slowest
 
 [testenv:py27-postgresql]
 commands =
-    {[testenv]commands}
-    {toxinidir}/tools/run_pifpaf.sh '{posargs}'
+  {[testenv]commands}
+  {toxinidir}/tools/run_pifpaf.sh '{posargs}'
 
 [testenv:py35]
 commands =
-    {[testenv]commands}
-    stestr run {posargs}
-    stestr slowest
+  {[testenv]commands}
+  stestr run {posargs}
+  stestr slowest
 
 [testenv:py35-postgresql]
 commands =
@@ -41,6 +41,8 @@ commands =
     {toxinidir}/tools/run_pifpaf.sh '{posargs}'
 
 [testenv:functional]
+# Always run functional tests using Python 3.
+basepython=python3.5
 setenv = VIRTUAL_ENV={envdir}
          OS_TEST_PATH=./deckhand/tests/functional
          LANGUAGE=en_US