From 847f41c2757107089b4360cb96bc3e14dd4b5bdc Mon Sep 17 00:00:00 2001 From: Sumit Naiksatam Date: Sat, 18 Mar 2017 13:46:08 -0700 Subject: [PATCH] Shared column in APG schema & change devstack plugin name Also updates exercise script which will catch this. And also fixes an issue due to which the AIM gate job was running against the master branch of GBP source versus the branch checked out for this patchset by the infra job. This is fixed by changing the GBP devstack plugin name to group-based-policy instead of the earlier name 'gbp'. This allows the job to use the 'group-based-policy' source directory cloned by the OpenStack infra job (for the current patchset) as opposed to cloning into a new 'gbp' directory from the master. Unfortunately, a lot of special casing needs to be introduced for configuration of other services as well. It is not possible to get away from this special casing to be able to install from the intree devstack plugin and the GBP master branch is behind the OpenStack master release. Closes-bug: 1674024 Change-Id: I5bd3f1c3ecfbedbd24243c2c111472dcff9059a2 --- devstack/README-NFP-Base-Mode | 10 +++++----- devstack/lib/{gbp => group-based-policy} | 4 ++-- devstack/local.conf.nfp | 2 +- devstack/plugin.sh | 12 +++++------ devstack/settings | 20 +++++++++++-------- doc/source/installation.rst | 19 +++++++++++++----- .../b6e301d5757f_application_policy_group.py | 3 +++ .../contrib/devstack/exercises-aim/gbp_aim.sh | 11 +++++++--- .../tests/contrib/devstack/exercises/gbp.sh | 11 +++++++--- .../tests/contrib/devstack/local-aim.conf | 7 +------ gbpservice/tests/contrib/functions-gbp | 15 ++++++++++++++ .../contrib/gbpfunctests/libs/verify_libs.py | 13 ++++++++++++ gbpservice/tests/contrib/post_test_hook.sh | 6 ++++++ 13 files changed, 94 insertions(+), 39 deletions(-) rename devstack/lib/{gbp => group-based-policy} (98%) diff --git a/devstack/README-NFP-Base-Mode b/devstack/README-NFP-Base-Mode index 507393988..d1a9ca8d3 100644 --- a/devstack/README-NFP-Base-Mode +++ b/devstack/README-NFP-Base-Mode @@ -32,7 +32,7 @@ Steps to test Base Mode: ======================== (1) Create a test chain with the service VM. - # cd /opt/stack/gbp/devstack/exercises/nfp_service + # cd /opt/stack/group-based-policy/devstack/exercises/nfp_service # bash lb_base.sh (2) Login to the UI, create one member in consumer group and two members @@ -45,14 +45,14 @@ Steps to test Base Mode: (4) Delete members created in the consumer and provider groups in step 2. (5) Delete the test chain. - # cd /opt/stack/gbp/devstack/exercises/nfp_service + # cd /opt/stack/group-based-policy/devstack/exercises/nfp_service # bash lb_base_clean.sh Steps to test Base Mode with VM: ================================ (1) Create a test chain with the service VM. - # cd /opt/stack/gbp/devstack/exercises/nfp_service + # cd /opt/stack/group-based-policy/devstack/exercises/nfp_service # bash fw_base_vm.sh (2) Login to the UI, create a member in the consumer and provider groups. @@ -60,12 +60,12 @@ Steps to test Base Mode with VM: (3) Test firewall with traffic from consumer VM. (4) Log-in into service VM - # cd /opt/stack/gbp/gbpservice/contrib/nfp/tools/image_builder + # cd /opt/stack/group-based-policy/gbpservice/contrib/nfp/tools/image_builder # sudo ip netns exec nfp-proxy ssh -i nfp_reference_service ubuntu@ (5) Delete members created in the consumer and provider groups in step 2. (6) Delete the test chain. - # cd /opt/stack/gbp/devstack/exercises/nfp_service + # cd /opt/stack/group-based-policy/devstack/exercises/nfp_service # bash fw_base_vm_clean.sh diff --git a/devstack/lib/gbp b/devstack/lib/group-based-policy similarity index 98% rename from devstack/lib/gbp rename to devstack/lib/group-based-policy index cec003d93..525376e9c 100755 --- a/devstack/lib/gbp +++ b/devstack/lib/group-based-policy @@ -1,4 +1,4 @@ -# lib/gbp +# lib/group-based-policy # functions - functions specific to group-based-policy # Dependencies: @@ -15,7 +15,7 @@ # ``unstack.sh`` calls the entry points in this order: # Set up default directories -GBPSERVICE_DIR=$DEST/gbp +GBPSERVICE_DIR=$DEST/group-based-policy GBPCLIENT_DIR=$DEST/python-gbpclient GBPHEAT_DIR=$DEST/gbpautomation GBPUI_DIR=$DEST/gbpui diff --git a/devstack/local.conf.nfp b/devstack/local.conf.nfp index 588792459..8391b9ad8 100644 --- a/devstack/local.conf.nfp +++ b/devstack/local.conf.nfp @@ -55,5 +55,5 @@ if [[ $ENABLE_NFP = True ]]; then fi fi -enable_plugin gbp $GBPSERVICE_REPO $GBPSERVICE_BRANCH +enable_plugin group-based-policy $GBPSERVICE_REPO $GBPSERVICE_BRANCH diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 020c687b9..22a0fe9b7 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -104,8 +104,8 @@ if is_service_enabled group-policy; then fi elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then echo_summary "Configuring $GBP" - gbp_configure_nova - gbp_configure_heat + [[ $ENABLE_APIC_AIM_GATE = False ]] && gbp_configure_nova + [[ $ENABLE_APIC_AIM_GATE = False ]] && gbp_configure_heat gbp_configure_neutron if [[ $ENABLE_NFP = True ]]; then echo_summary "Configuring $NFP" @@ -123,11 +123,11 @@ if is_service_enabled group-policy; then [[ $ENABLE_NFP = True ]] && install_nfpgbpservice init_gbpservice [[ $ENABLE_NFP = True ]] && init_nfpgbpservice - install_gbpheat - install_gbpui + [[ $ENABLE_APIC_AIM_GATE = False ]] && install_gbpheat + [[ $ENABLE_APIC_AIM_GATE = False ]] && install_gbpui [[ $ENABLE_APIC_AIM = True || $ENABLE_APIC_AIM_GATE = True ]] && configure_apic_aim - stop_apache_server - start_apache_server + [[ $ENABLE_APIC_AIM_GATE = False ]] && stop_apache_server + [[ $ENABLE_APIC_AIM_GATE = False ]] && start_apache_server elif [[ "$1" == "stack" && "$2" == "extra" ]]; then echo_summary "Initializing $GBP" if [[ $ENABLE_NFP = True ]]; then diff --git a/devstack/settings b/devstack/settings index c062e60d8..405cf232e 100755 --- a/devstack/settings +++ b/devstack/settings @@ -1,14 +1,14 @@ -# Make sure the plugin name in local.conf is "gbp", as in: enable_plugin gbp -source $DEST/gbp/devstack/lib/gbp +# Make sure the plugin name in local.conf is "group-based-policy", as in: enable_plugin group-based-policy +source $DEST/group-based-policy/devstack/lib/group-based-policy ENABLE_APIC_AIM=${ENABLE_APIC_AIM:-False} ENABLE_APIC_AIM_GATE=${ENABLE_APIC_AIM_GATE:-False} -[[ $ENABLE_APIC_AIM = True ]] && source $DEST/gbp/devstack/lib/apic_aim -[[ $ENABLE_APIC_AIM_GATE = True ]] && source $DEST/gbp/devstack/lib/apic_aim +[[ $ENABLE_APIC_AIM = True ]] && source $DEST/group-based-policy/devstack/lib/apic_aim +[[ $ENABLE_APIC_AIM_GATE = True ]] && source $DEST/group-based-policy/devstack/lib/apic_aim ENABLE_NFP=${ENABLE_NFP:-False} [[ $ENABLE_NFP = True ]] && NFP_DEVSTACK_MODE=${NFP_DEVSTACK_MODE:-base} -[[ $ENABLE_NFP = True ]] && source $DEST/gbp/devstack/lib/nfp +[[ $ENABLE_NFP = True ]] && source $DEST/group-based-policy/devstack/lib/nfp # VM locations ConfiguratorQcow2Image=${ConfiguratorQcow2Image:-build} @@ -35,7 +35,6 @@ APICAPI_BRANCH=${APICAPI_BRANCH:-aci_integration_manager} # Enable necessary services, including group-policy (and disable others) disable_service n-net -enable_service n-novnc enable_service q-svc if [[ $ENABLE_APIC_AIM = True || $ENABLE_APIC_AIM_GATE = True ]]; then disable_service q-agt @@ -61,8 +60,13 @@ fi enable_service q-meta enable_service neutron enable_service group-policy -disable_service tempest -ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng +if [[ $ENABLE_APIC_AIM_GATE = False ]]; then + ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng + disable_service tempest +else + # this may not be required + DISABLED_SERVICES+=,n-api,n-cpu,n-cond,n-sch,n-novnc,n-cauth,g-api,g-reg,c-sch,c-api,c-vol,horizon,heat,h-api,h-api-cfn,h-api-cw,h-eng,dstat,tempest +fi if [[ $ENABLE_NFP = True ]]; then # NFP services enable_service nfp_orchestrator diff --git a/doc/source/installation.rst b/doc/source/installation.rst index e61f50e9f..7cfc5f81a 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -14,15 +14,25 @@ Or, if you have virtualenvwrapper installed:: Using DevStack -------------- -First, clone the latest ``stable/mitaka`` branch of DevStack:: +First, clone the latest ``stable/newton`` branch of DevStack:: - $ git clone -b stable/mitaka https://git.openstack.org/openstack-dev/devstack + $ git clone -b stable/newton https://git.openstack.org/openstack-dev/devstack $ cd devstack Then, create a basic ``local.conf`` including at least the following lines:: [[local|localrc]] - enable_plugin gbp https://git.openstack.org/openstack/group-based-policy master + enable_plugin group-based-policy https://git.openstack.org/openstack/group-based-policy master + +Or, if you need install from a patch under review:: + + [[local|localrc]] + enable_plugin group-based-policy https://git.openstack.org/openstack/group-based-policy + +where, GITREF is the patchset reference of the patchset under review. E.g.:: + + [[local|localrc]] + enable_plugin group-based-policy https://git.openstack.org/openstack/group-based-policy refs/changes/65/353265/2 Finally, you are ready to run ``stack.sh``. @@ -43,5 +53,4 @@ pointing to GitHub:: GIT_BASE=https://github.com RECLONE=True - enable_plugin gbp https://github.com/openstack/group-based-policy.git master - + enable_plugin group-based-policy https://github.com/openstack/group-based-policy.git master diff --git a/gbpservice/neutron/db/migration/alembic_migrations/versions/b6e301d5757f_application_policy_group.py b/gbpservice/neutron/db/migration/alembic_migrations/versions/b6e301d5757f_application_policy_group.py index 972e4d556..64b25c0f0 100644 --- a/gbpservice/neutron/db/migration/alembic_migrations/versions/b6e301d5757f_application_policy_group.py +++ b/gbpservice/neutron/db/migration/alembic_migrations/versions/b6e301d5757f_application_policy_group.py @@ -25,6 +25,7 @@ down_revision = 'daaa11a358a2' from alembic import op import sqlalchemy as sa +from sqlalchemy import sql def upgrade(): @@ -37,6 +38,8 @@ def upgrade(): sa.Column('description', sa.String(length=255), nullable=True), sa.Column('status', sa.String(length=16), nullable=True), sa.Column('status_details', sa.String(length=4096), nullable=True), + sa.Column('shared', sa.Boolean, nullable=True, + server_default=sql.false()), sa.PrimaryKeyConstraint('id')) op.add_column( diff --git a/gbpservice/tests/contrib/devstack/exercises-aim/gbp_aim.sh b/gbpservice/tests/contrib/devstack/exercises-aim/gbp_aim.sh index eedde5f3d..27b1097e5 100755 --- a/gbpservice/tests/contrib/devstack/exercises-aim/gbp_aim.sh +++ b/gbpservice/tests/contrib/devstack/exercises-aim/gbp_aim.sh @@ -68,10 +68,13 @@ gbp policy-rule-set-create icmp-policy-rule-set --policy-rules ping-policy-rule gbp policy-rule-set-create web-policy-rule-set --policy-rules web-policy-rule # ====== PROJECT OPERATION ====== +# APG creation +gbp apg-create myapp + # PTGs creation -gbp group-create web -gbp group-create client-1 -gbp group-create client-2 +gbp group-create --application-policy-group myapp web +gbp group-create --application-policy-group myapp client-1 +gbp group-create --application-policy-group myapp client-2 # PT creation WEB_PORT=$(gbp policy-target-create web-pt-1 --policy-target-group web | awk "/port_id/ {print \$4}") @@ -129,6 +132,8 @@ gbp group-delete web gbp group-delete client-1 gbp group-delete client-2 +gbp apg-delete myapp + gbp policy-rule-set-delete icmp-policy-rule-set gbp policy-rule-set-delete web-policy-rule-set diff --git a/gbpservice/tests/contrib/devstack/exercises/gbp.sh b/gbpservice/tests/contrib/devstack/exercises/gbp.sh index 1186b1438..bca056edd 100755 --- a/gbpservice/tests/contrib/devstack/exercises/gbp.sh +++ b/gbpservice/tests/contrib/devstack/exercises/gbp.sh @@ -68,10 +68,13 @@ gbp policy-rule-set-create icmp-policy-rule-set --policy-rules ping-policy-rule gbp policy-rule-set-create web-policy-rule-set --policy-rules web-policy-rule # ====== PROJECT OPERATION ====== +# APG creation +gbp apg-create myapp + # PTGs creation -gbp group-create web -gbp group-create client-1 -gbp group-create client-2 +gbp group-create --application-policy-group myapp web +gbp group-create --application-policy-group myapp client-1 +gbp group-create --application-policy-group myapp client-2 # PT creation WEB_PORT=$(gbp policy-target-create web-pt-1 --policy-target-group web | awk "/port_id/ {print \$4}") @@ -127,6 +130,8 @@ gbp group-delete web gbp group-delete client-1 gbp group-delete client-2 +gbp apg-delete myapp + gbp policy-rule-set-delete icmp-policy-rule-set gbp policy-rule-set-delete web-policy-rule-set diff --git a/gbpservice/tests/contrib/devstack/local-aim.conf b/gbpservice/tests/contrib/devstack/local-aim.conf index 5557067c8..777a3ad0c 100644 --- a/gbpservice/tests/contrib/devstack/local-aim.conf +++ b/gbpservice/tests/contrib/devstack/local-aim.conf @@ -12,12 +12,7 @@ SCREEN_LOGDIR=$DEST/logs/screen LOGFILE=$DEST/logs/stack.sh.log SKIP_EXERCISES=volumes,trove,swift,sahara,euca,bundle,boot_from_volume,aggregates,zaqar,client-env,client-args,sec_groups,neutron-adv-test,floating_ips,horizon -#OFFLINE=True -RECLONE=True - -# TODO(Sumit): Revert the following once this patch is merged -#enable_plugin gbp https://github.com/openstack/group-based-policy.git master -enable_plugin gbp https://git.openstack.org/openstack/group-based-policy refs/changes/47/439247/16 +enable_plugin group-based-policy https://github.com/openstack/group-based-policy.git master ENABLE_APIC_AIM_GATE=True diff --git a/gbpservice/tests/contrib/functions-gbp b/gbpservice/tests/contrib/functions-gbp index b56e20fb7..d1c0e2416 100644 --- a/gbpservice/tests/contrib/functions-gbp +++ b/gbpservice/tests/contrib/functions-gbp @@ -50,6 +50,20 @@ function prepare_gbp_aim_devstack { # Use the aim version of the shared PRS test sudo mv $GBP_FUNC_DIR/testcases/tc_gbp_prs_pr_shared_func.py.aim $GBP_FUNC_DIR/testcases/tc_gbp_prs_pr_shared_func.py sudo mv $GBP_FUNC_DIR/testcases/tc_gbp_prs_func.py.aim $GBP_FUNC_DIR/testcases/tc_gbp_prs_func.py + # The following should updated when master moves to a new release + # We need to do the following since the infra job clones these repos and + # checks out the master branch (as this is the master branch) and later + # does not switch to the stable/newton branch when installing devstack + # since the repo is already present. + # This can be worked around by changing the job description in + # project-config to set BRANCH_OVERRIDE to use the stable/newton branch + sudo git --git-dir=/opt/stack/new/neutron/.git --work-tree=/opt/stack/new/neutron checkout stable/newton + sudo git --git-dir=/opt/stack/new/nova/.git --work-tree=/opt/stack/new/nova checkout stable/newton + sudo git --git-dir=/opt/stack/new/keystone/.git --work-tree=/opt/stack/new/keystone checkout stable/newton + sudo git --git-dir=/opt/stack/new/cinder/.git --work-tree=/opt/stack/new/cinder checkout stable/newton + sudo git --git-dir=/opt/stack/new/requirements/.git --work-tree=/opt/stack/new/requirements checkout stable/newton + export ENABLED_SERVICES="key,rabbit,mysql" + source $TOP_DIR/functions source $TOP_DIR/functions-common } @@ -105,6 +119,7 @@ function check_residual_resources { gbp nat-pool-list gbp external-policy-list gbp external-segment-list + gbp apg-list neutron router-list neutron net-list diff --git a/gbpservice/tests/contrib/gbpfunctests/libs/verify_libs.py b/gbpservice/tests/contrib/gbpfunctests/libs/verify_libs.py index 916890292..fa4d0b058 100755 --- a/gbpservice/tests/contrib/gbpfunctests/libs/verify_libs.py +++ b/gbpservice/tests/contrib/gbpfunctests/libs/verify_libs.py @@ -23,6 +23,19 @@ _log = logging.getLogger() _log.setLevel(logging.INFO) +orig_getoutput = commands.getoutput + + +def getoutput(cmd): + _log.info('Running cmd: %s\n' % (cmd)) + cmd_out = orig_getoutput(cmd) + _log.info('Cmd output: %s\n' % (cmd_out)) + return cmd_out + + +commands.getoutput = getoutput + + class Gbp_Verify(object): def __init__(self): diff --git a/gbpservice/tests/contrib/post_test_hook.sh b/gbpservice/tests/contrib/post_test_hook.sh index 57ffdad4d..69e8f2f63 100644 --- a/gbpservice/tests/contrib/post_test_hook.sh +++ b/gbpservice/tests/contrib/post_test_hook.sh @@ -6,6 +6,12 @@ set -x trap prepare_logs ERR +sudo git --git-dir=/opt/stack/new/neutron/.git --work-tree=/opt/stack/new/neutron show --name-only +sudo git --git-dir=/opt/stack/new/neutron/.git --work-tree=/opt/stack/new/neutron status +sudo pip show neutron-lib +sudo git --git-dir=/opt/stack/new/group-based-policy/.git --work-tree=/opt/stack/new/group-based-policy show --name-only +sudo git --git-dir=/opt/stack/new/group-based-policy/.git --work-tree=/opt/stack/new/group-based-policy status + # Run exercise scripts $TOP_DIR/exercise.sh exercises_exit_code=$?