openstack/storlets
Code Issues Proposed changes

Drop Python 2 support in middleware/gateway

Browse Source 
... because Swift is finally removing its support for Python 2.

Change-Id: I1913e7ceeaf06f7bd2ae1d69d5d1ecf8ddc71e35
This commit is contained in:
Takashi Kajinami 2022-03-16 09:45:28 +09:00
parent 132d3509f2
commit 302b73ca17
12 changed files with 18 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.functests
View File

@ -16,14 +16,6 @@ shift
ant build_storlets
if [ ${USE_PYTHON3} == "False" ]; then
# TODO(takashi): We should use python-swiftclent 3.8.1 to avoid connection
# leak on py2.7(lp#1873435). Remove this once we get a fixed
# version released.
pip uninstall python-swiftclient
pip install python-swiftclient==3.8.1
fi
SRC_DIR=$(cd $(dirname $0); pwd)
cd ${SRC_DIR}/tests/functional
if [ "$FLAVOR" == "jenkins" ]; then

16
.zuul.yaml
View File

@ -1,17 +1,14 @@
- project:
templates:
- openstack-python-jobs
- openstack-python3-zed-jobs
- publish-openstack-docs-pti
- release-notes-jobs-python3
check:
jobs:
- storlets-functional
- storlets-functional-py3
gate:
jobs:
- storlets-functional
- storlets-functional-py3
- job:
name: storlets-functional
@ -20,19 +17,6 @@
post-run: playbooks/storlets-functional/post.yaml
nodeset: ubuntu-focal
timeout: 2400
voting: false
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- ^tests/unit/.*$
- job:
name: storlets-functional-py3
pre-run: playbooks/storlets-functional/pre-py3.yaml
run: playbooks/storlets-functional/run-py3.yaml
post-run: playbooks/storlets-functional/post-py3.yaml
nodeset: ubuntu-focal
timeout: 2400
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$

18
devstack/localrc.py2.sample
View File

@ -1,18 +0,0 @@
ENABLE_HTTPD_MOD_WSGI_SERVICES=False
ENABLED_SERVICES=key,swift,mysql
# We need stable/train keystone to run it on python2
KEYSTONE_BRANCH=stable/train
HOST_IP=127.0.0.1
ADMIN_PASSWORD=admin
MYSQL_PASSWORD=$ADMIN_PASSWORD
RABBIT_PASSWORD=$ADMIN_PASSWORD
SERVICE_PASSWORD=$ADMIN_PASSWORD
SWIFT_SERVICE_PROTOCOL=${SWIFT_SERVICE_PROTOCOL:-http}
SWIFT_DEFAULT_BIND_PORT=${SWIFT_DEFAULT_BIND_PORT:-8080}
# service local host is used for ring building
SWIFT_SERVICE_LOCAL_HOST=$HOST_IP
# service listen address for prox
SWIFT_SERVICE_LISTEN_ADDRESS=$HOST_IP
SWIFT_LOOPBACK_DISK_SIZE=20G
SWIFT_MAX_FILE_SIZE=5368709122
SWIFT_HASH=1234567890

4
devstack/plugin.sh
View File

@ -125,11 +125,7 @@ function configure_swift_and_keystone_for_storlets {
_generate_swift_middleware_conf
_generate_storlet-docker-gateway
if [ "${USE_PYTHON3}" == "False" ]; then
sudo python2 devstack/swift_config.py install /tmp/swift_middleware_conf $STORLETS_SWIFT_RUNTIME_USER
else
sudo python3 devstack/swift_config.py install /tmp/swift_middleware_conf $STORLETS_SWIFT_RUNTIME_USER
fi
rm /tmp/swift_middleware_conf
rm /tmp/storlet-docker-gateway.conf

52
playbooks/storlets-functional/post-py3.yaml
View File

@ -1,52 +0,0 @@
---
- hosts: all
tasks:
- name: Create log directory to pull
file:
path: '{{ zuul.project.src_dir }}/logs'
state: directory
- name: Prepare the logs into user space
shell: "egrep {{ item }} /var/log/syslog > {{ zuul.project.src_dir }}/logs/{{ item }}.log"
with_items:
- "account-server"
- "container-server"
- "object-server"
- "proxy-server"
- "ansible"
- "storlets-daemon"
- "daemon-factory"
become: yes
- name: Change log permisson as readable
file:
path: "{{ zuul.project.src_dir }}/logs/{{ item }}.log"
mode: 0644
with_items:
- "account-server"
- "container-server"
- "object-server"
- "proxy-server"
- "ansible"
- "storlets-daemon"
- "daemon-factory"
become: yes
- name: Copy logs from worker nodes to executor node
synchronize:
# TODO: make it sort of tox_envdir
src: '{{ zuul.project.src_dir }}/logs'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
ignore_errors: yes
- name: Copy conf files from worker nodes to executor node
synchronize:
src: '/etc/swift'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
ignore_errors: yes

24
playbooks/storlets-functional/pre-py3.yaml
View File

@ -1,24 +0,0 @@
---
- hosts: all
tasks:
- name: installing python3 packages
package: name={{ item }} state=present
with_items:
- python3
- python3-venv
- python3-nose
- python3-pip
become: yes
- shell:
cmd: |
set -e
set -x
./s2aio.sh install
executable: /bin/bash
chdir: 'src/{{ zuul.project.canonical_name }}'
environment:
USE_PYTHON3: "True"
roles:
- ensure-tox

11
playbooks/storlets-functional/pre.yaml
View File

@ -1,6 +1,15 @@
---
- hosts: all
tasks:
- name: installing python3 packages
package: name={{ item }} state=present
with_items:
- python3
- python3-venv
- python3-nose
- python3-pip
become: yes
- shell:
cmd: |
set -e
@ -8,8 +17,6 @@
./s2aio.sh install
executable: /bin/bash
chdir: 'src/{{ zuul.project.canonical_name }}'
environment:
USE_PYTHON3: "False"
roles:
- ensure-tox

6
playbooks/storlets-functional/run-py3.yaml
View File

@ -1,6 +0,0 @@
---
- hosts: all
roles:
- tox
vars:
- tox_envlist: func-py3

4
playbooks/storlets-functional/run.yaml
View File

@ -3,6 +3,4 @@
roles:
- tox
vars:
- tox_envlist: func-py3
environment:
USE_PYTHON3: "False"
- tox_envlist: func

6
s2aio.sh
View File

@ -33,13 +33,9 @@ function usage {
function _prepare_devstack_env {
# Checkout devstack
if [ ! -e $DEVSTACK_DIR ]; then
git clone https://github.com/openstack/devstack.git $DEVSTACK_DIR
if [ ${USE_PYTHON3} == "False" ]; then
cp devstack/localrc.py2.sample $DEVSTACK_DIR/localrc
else
git clone https://github.com/openstack-dev/devstack.git $DEVSTACK_DIR
cp devstack/localrc.sample $DEVSTACK_DIR/localrc
fi
fi
source $DEVSTACK_DIR/stackrc
source $DEVSTACK_DIR/functions

4
setup.cfg
View File

@ -6,6 +6,7 @@ description_file =
author = OpenStack
author_email = openstack-discuss@lists.openstack.org
home_page = https://docs.openstack.org/storlets/latest/
python_requires = >=3.6
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
@ -13,13 +14,12 @@ classifier =
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3 :: Only
[pbr]
skip_authors = True

32
tox.ini
View File

@ -1,11 +1,12 @@
[tox]
minversion = 3.2.0
envlist = py27,py36,py38,py39,pep8
envlist = py36,py38,py39,pep8
skipsdist = True
ignore_basepython_conflict = True
[testenv]
basepython = python3
usedevelop = True
setenv =
VIRTUAL_ENV={envdir}
@ -27,46 +28,24 @@ whitelist_externals = bash
rm
[testenv:pep8python]
basepython = python3
commands =
flake8
flake8 --filename=* bin --exclude=bin/*.sh
[testenv:py27]
[testenv:func]
deps =
-c{toxinidir}/py2-constraints.txt
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
git+https://github.com/openstack/swift.git
[testenv:func]
basepython = python2.7
deps = {[testenv:py27]deps}
setenv =
VIRTUAL_ENV={envdir}
STORLET_SAMPLE_PATH={toxinidir}/StorletSamples
CLUSTER_CONF_DIR={toxinidir}
commands = {toxinidir}/.functests jenkins
passenv = USE_PYTHON3
[testenv:func-py3]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
git+https://github.com/openstack/swift.git
setenv =
VIRTUAL_ENV={envdir}
STORLET_SAMPLE_PATH={toxinidir}/StorletSamples
CLUSTER_CONF_DIR={toxinidir}
commands = {[testenv:func]commands}
passenv = {[testenv:func]passenv}
[testenv:venv]
basepython = python3
commands = {posargs}
[testenv:cover]
basepython = python3
setenv =
{[testenv]setenv}
PYTHON=coverage run --source storlets --parallel-mode
@ -77,7 +56,6 @@ commands =
coverage xml -o cover/coverage.xml
[testenv:docs]
basepython = python3
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/requirements.txt
@ -87,11 +65,9 @@ commands =
sphinx-build -a -W -E -b html doc/source doc/build/html
[testenv:debug]
basepython = python3
commands = oslo_debug_helper {posargs}
[testenv:bashate]
basepython = python3
# Run bashate check for all bash scripts
# Ignores the following rules:
# E003: Indent not multiple of 4 (we prefer to use multiples of 2)
@ -117,7 +93,6 @@ builtins = _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build
[testenv:pep8]
basepython = python3
commands =
{[testenv:pep8python]commands}
{[testenv:bashate]commands}
@ -125,6 +100,5 @@ passenv =
HOME
[testenv:releasenotes]
basepython = python3
deps = {[testenv:docs]deps}
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html