From 4bc703883e6a17c817c07dc803cb0fef7f54a814 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Mon, 13 Mar 2017 10:05:21 -0400 Subject: [PATCH] Remove MySQL Even though there is nothing to read from secure.conf anymore, it is kept around intact since we may want to use this for ZooKeeper credentials at some point. Change-Id: Ieb3a93b09c889f74da3463494957335aaaa9f40f --- README.rst | 29 --- bindep.txt | 2 - devstack/plugin.sh | 18 +- doc/source/configuration.rst | 19 +- doc/source/installation.rst | 3 +- nodepool/cmd/config_validator.py | 1 - nodepool/config.py | 4 +- nodepool/nodedb.py | 247 -------------------- nodepool/tests/__init__.py | 50 +--- nodepool/tests/fixtures/secure.conf | 3 +- requirements.txt | 1 - tools/fake-secure.conf | 3 +- tools/test-setup.sh | 33 --- tools/zuul-nodepool-integration/secure.conf | 3 +- 14 files changed, 10 insertions(+), 406 deletions(-) delete mode 100644 nodepool/nodedb.py delete mode 100755 tools/test-setup.sh diff --git a/README.rst b/README.rst index 4a99ffbc3..b4d55639e 100644 --- a/README.rst +++ b/README.rst @@ -47,29 +47,6 @@ If the cloud being used has no default_floating_pool defined in nova.conf, you will need to define a pool name using the nodepool yaml file to use floating ips. - -Set up database for interactive testing: - -.. code-block:: bash - - mysql -u root - - mysql> create database nodepool; - mysql> GRANT ALL ON nodepool.* TO 'nodepool'@'localhost'; - mysql> flush privileges; - -Set up database for unit tests: - -.. code-block:: bash - - mysql -u root - mysql> grant all privileges on *.* to 'openstack_citest'@'localhost' identified by 'openstack_citest' with grant option; - mysql> flush privileges; - mysql> create database openstack_citest; - -Note that the script tools/test-setup.sh can be used for the step -above. - Export variable for your ssh key so you can log into the created instances: .. code-block:: bash @@ -92,9 +69,3 @@ Use the following tool to check on progress: .. code-block:: bash nodepool image-list - -After each run (the fake nova provider is only in-memory): - -.. code-block:: bash - - mysql> delete from snapshot_image; delete from node; diff --git a/bindep.txt b/bindep.txt index 426c5db61..81d5723a5 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,8 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; # see http://docs.openstack.org/infra/bindep/ for additional information. -mysql-client [test] -mysql-server [test] python-dev [platform:dpkg test] python-devel [platform:rpm test] zookeeperd [platform:dpkg test] diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7149208e0..0f68b1be0 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -101,7 +101,6 @@ EOF function nodepool_write_config { sudo mkdir -p $(dirname $NODEPOOL_CONFIG) sudo mkdir -p $(dirname $NODEPOOL_SECURE) - local dburi=$(database_connection_url nodepool) cat > /tmp/logging.conf < /tmp/secure.conf << EOF -[database] -# The mysql password here may be different depending on your -# devstack install, you should double check it (the devstack var -# is MYSQL_PASSWORD and if unset devstack should prompt you for -# the value). -dburi: $dburi +# Empty EOF sudo mv /tmp/secure.conf $NODEPOOL_SECURE @@ -174,11 +168,6 @@ EOF # example element. elements-dir: $(dirname $NODEPOOL_CONFIG)/elements images-dir: $NODEPOOL_DIB_BASE_PATH/images -# The mysql password here may be different depending on your -# devstack install, you should double check it (the devstack var -# is MYSQL_PASSWORD and if unset devstack should prompt you for -# the value). -dburi: '$dburi' zookeeper-servers: - host: localhost @@ -378,7 +367,6 @@ EOF mkdir -p $HOME/.cache/openstack/ } -# Initialize database # Create configs # Setup custom flavor function configure_nodepool { @@ -390,10 +378,6 @@ function configure_nodepool { # write the elements nodepool_write_elements - - # builds a fresh db - recreate_database nodepool - } function start_nodepool { diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index e41d4a80f..017d15751 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -4,23 +4,8 @@ Configuration ============= Nodepool reads its secure configuration from ``/etc/nodepool/secure.conf`` -by default. The secure file is a standard ini config file, with -one section for the database. - - [database] - dburi={dburi} - -Following settings are available:: - -**required** - - ``dburi`` - Indicates the URI for the database connection. See the `SQLAlchemy - documentation - `_ - for the syntax. Example:: - - dburi='mysql+pymysql://nodepool@localhost/nodepool' +by default. The secure file is a standard ini config file. Note that this +file is currently unused, but may be in the future. Nodepool reads its configuration from ``/etc/nodepool/nodepool.yaml`` by default. The configuration file follows the standard YAML syntax diff --git a/doc/source/installation.rst b/doc/source/installation.rst index fc9f25cfc..040c748d0 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,8 +55,7 @@ Configuration Nodepool has two required configuration files: secure.conf and nodepool.yaml, and an optional logging configuration file logging.conf. The secure.conf file is used to store nodepool configurations that contain -sensitive data, such as the Nodepool database password and Jenkins -api key. The nodepool.yaml files is used to store all other +sensitive data. The nodepool.yaml files is used to store all other configurations. The logging configuration file is in the standard python logging diff --git a/nodepool/cmd/config_validator.py b/nodepool/cmd/config_validator.py index f8b463762..285ac7da2 100644 --- a/nodepool/cmd/config_validator.py +++ b/nodepool/cmd/config_validator.py @@ -105,7 +105,6 @@ class ConfigValidator: top_level = { 'elements-dir': str, 'images-dir': str, - 'dburi': str, 'zookeeper-servers': [{ 'host': str, 'port': int, diff --git a/nodepool/config.py b/nodepool/config.py index aa77cc11e..ed5f1602b 100644 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -131,12 +131,10 @@ def loadConfig(config_path): newconfig = Config() newconfig.db = None - newconfig.dburi = None newconfig.providers = {} newconfig.labels = {} newconfig.elementsdir = config.get('elements-dir') newconfig.imagesdir = config.get('images-dir') - newconfig.dburi = None newconfig.provider_managers = {} newconfig.zookeeper_servers = {} newconfig.diskimages = {} @@ -277,7 +275,7 @@ def loadSecureConfig(config, secure_config_path): secure = ConfigParser.ConfigParser() secure.readfp(open(secure_config_path)) - config.dburi = secure.get('database', 'dburi') + #config.dburi = secure.get('database', 'dburi') def _cloudKwargsFromProvider(provider): diff --git a/nodepool/nodedb.py b/nodepool/nodedb.py deleted file mode 100644 index d7ae959b7..000000000 --- a/nodepool/nodedb.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright (C) 2011-2014 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -# States: -# The cloud provider is building this machine. We have an ID, but it's -# not ready for use. -BUILDING = 1 -# The machine is ready for use. -READY = 2 -# This can mean in-use, or used but complete. -USED = 3 -# Delete this machine immediately. -DELETE = 4 -# Keep this machine indefinitely. -HOLD = 5 -# Acceptance testing (pre-ready) -TEST = 6 - - -STATE_NAMES = { - BUILDING: 'building', - READY: 'ready', - USED: 'used', - DELETE: 'delete', - HOLD: 'hold', - TEST: 'test', - } - -from sqlalchemy import Table, Column, Integer, String, \ - MetaData, create_engine -from sqlalchemy.orm import scoped_session, mapper -from sqlalchemy.orm.session import Session, sessionmaker - -metadata = MetaData() - -node_table = Table( - 'node', metadata, - Column('id', Integer, primary_key=True), - Column('provider_name', String(255), index=True, nullable=False), - Column('label_name', String(255), index=True, nullable=False), - Column('target_name', String(255), index=True, nullable=False), - Column('manager_name', String(255)), - # Machine name - Column('hostname', String(255), index=True), - # Eg, jenkins node name - Column('nodename', String(255), index=True), - # Provider assigned id for this machine - Column('external_id', String(255)), - # Provider availability zone for this machine - Column('az', String(255)), - # Primary IP address - Column('ip', String(255)), - # Internal/fixed IP address - Column('ip_private', String(255)), - # One of the above values - Column('state', Integer), - # Time of last state change - Column('state_time', Integer), - # Comment about the state of the node - used to annotate held nodes - Column('comment', String(255)), - mysql_engine='InnoDB', - ) -job_table = Table( - 'job', metadata, - Column('id', Integer, primary_key=True), - # The name of the job - Column('name', String(255), index=True), - # Automatically hold up to this number of nodes that fail this job - Column('hold_on_failure', Integer), - mysql_engine='InnoDB', - ) - - -class Node(object): - def __init__(self, provider_name, label_name, target_name, az, - hostname=None, external_id=None, ip=None, ip_private=None, - manager_name=None, state=BUILDING, comment=None): - self.provider_name = provider_name - self.label_name = label_name - self.target_name = target_name - self.manager_name = manager_name - self.external_id = external_id - self.az = az - self.ip = ip - self.ip_private = ip_private - self.hostname = hostname - self.state = state - self.comment = comment - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - @property - def state(self): - return self._state - - @state.setter - def state(self, state): - self._state = state - self.state_time = int(time.time()) - session = Session.object_session(self) - if session: - session.commit() - - -class Job(object): - def __init__(self, name=None, hold_on_failure=0): - self.name = name - self.hold_on_failure = hold_on_failure - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - -mapper(Job, job_table) - - -mapper(Node, node_table, - properties=dict( - _state=node_table.c.state)) - - -class NodeDatabase(object): - def __init__(self, dburi): - engine_kwargs = dict(echo=False, pool_recycle=3600) - if 'sqlite:' not in dburi: - engine_kwargs['max_overflow'] = -1 - - self.engine = create_engine(dburi, **engine_kwargs) - metadata.create_all(self.engine) - self.session_factory = sessionmaker(bind=self.engine) - self.session = scoped_session(self.session_factory) - - def getSession(self): - return NodeDatabaseSession(self.session) - - -class NodeDatabaseSession(object): - def __init__(self, session): - self.session = session - - def __enter__(self): - return self - - def __exit__(self, etype, value, tb): - if etype: - self.session().rollback() - else: - self.session().commit() - self.session().close() - self.session = None - - def abort(self): - self.session().rollback() - - def commit(self): - self.session().commit() - - def delete(self, obj): - self.session().delete(obj) - - def getNodes(self, provider_name=None, label_name=None, target_name=None, - state=None): - exp = self.session().query(Node).order_by( - node_table.c.provider_name, - node_table.c.label_name) - if provider_name: - exp = exp.filter_by(provider_name=provider_name) - if label_name: - exp = exp.filter_by(label_name=label_name) - if target_name: - exp = exp.filter_by(target_name=target_name) - if state: - exp = exp.filter(node_table.c.state == state) - return exp.all() - - def createNode(self, *args, **kwargs): - new = Node(*args, **kwargs) - self.session().add(new) - self.commit() - return new - - def getNode(self, id): - nodes = self.session().query(Node).filter_by(id=id).all() - if not nodes: - return None - return nodes[0] - - def getNodeByHostname(self, hostname): - nodes = self.session().query(Node).filter_by(hostname=hostname).all() - if not nodes: - return None - return nodes[0] - - def getNodeByNodename(self, nodename): - nodes = self.session().query(Node).filter_by(nodename=nodename).all() - if not nodes: - return None - return nodes[0] - - def getNodeByExternalID(self, provider_name, external_id): - nodes = self.session().query(Node).filter_by( - provider_name=provider_name, - external_id=external_id).all() - if not nodes: - return None - return nodes[0] - - def getJob(self, id): - jobs = self.session().query(Job).filter_by(id=id).all() - if not jobs: - return None - return jobs[0] - - def getJobByName(self, name): - jobs = self.session().query(Job).filter_by(name=name).all() - if not jobs: - return None - return jobs[0] - - def getJobs(self): - return self.session().query(Job).all() - - def createJob(self, *args, **kwargs): - new = Job(*args, **kwargs) - self.session().add(new) - self.commit() - return new diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index 8317c36d5..3d8cb6d2a 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -18,17 +18,14 @@ import glob import logging import os -import pymysql import random import string import subprocess import threading import tempfile import time -import uuid import fixtures -import lockfile import kazoo.client import testtools @@ -230,47 +227,6 @@ class RoundRobinTestCase(object): self.allocations)) -class MySQLSchemaFixture(fixtures.Fixture): - def setUp(self): - super(MySQLSchemaFixture, self).setUp() - - random_bits = ''.join(random.choice(string.ascii_lowercase + - string.ascii_uppercase) - for x in range(8)) - self.name = '%s_%s' % (random_bits, os.getpid()) - self.passwd = uuid.uuid4().hex - lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile') - with lock: - db = pymysql.connect(host="localhost", - user="openstack_citest", - passwd="openstack_citest", - db="openstack_citest") - cur = db.cursor() - cur.execute("create database %s" % self.name) - cur.execute( - "grant all on %s.* to '%s'@'localhost' identified by '%s'" % - (self.name, self.name, self.passwd)) - cur.execute("flush privileges") - - self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name, - self.passwd, - self.name) - self.addDetail('dburi', testtools.content.text_content(self.dburi)) - self.addCleanup(self.cleanup) - - def cleanup(self): - lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile') - with lock: - db = pymysql.connect(host="localhost", - user="openstack_citest", - passwd="openstack_citest", - db="openstack_citest") - cur = db.cursor() - cur.execute("drop database %s" % self.name) - cur.execute("drop user '%s'@'localhost'" % self.name) - cur.execute("flush privileges") - - class BuilderFixture(fixtures.Fixture): def __init__(self, configfile, cleanup_interval): super(BuilderFixture, self).__init__() @@ -296,9 +252,6 @@ class DBTestCase(BaseTestCase): def setUp(self): super(DBTestCase, self).setUp() self.log = logging.getLogger("tests") - f = MySQLSchemaFixture() - self.useFixture(f) - self.dburi = f.dburi self.secure_conf = self._setup_secure() self.setupZK() @@ -333,7 +286,8 @@ class DBTestCase(BaseTestCase): (fd, path) = tempfile.mkstemp() with open(configfile) as conf_fd: config = conf_fd.read() - os.write(fd, config.format(dburi=self.dburi)) + os.write(fd, config) + #os.write(fd, config.format(dburi=self.dburi)) os.close(fd) return path diff --git a/nodepool/tests/fixtures/secure.conf b/nodepool/tests/fixtures/secure.conf index 08f7d8909..b7db25411 100644 --- a/nodepool/tests/fixtures/secure.conf +++ b/nodepool/tests/fixtures/secure.conf @@ -1,2 +1 @@ -[database] -dburi={dburi} +# Empty diff --git a/requirements.txt b/requirements.txt index a11843862..53ebc2e75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ python-daemon>=2.0.4,<2.1.0 extras statsd>=3.0 sqlalchemy>=0.8.2,<1.1.0 -PyMySQL PrettyTable>=0.6,<0.8 # shade has a looser requirement on six than nodepool, so install six first six>=1.7.0 diff --git a/tools/fake-secure.conf b/tools/fake-secure.conf index d29d9c094..b7db25411 100644 --- a/tools/fake-secure.conf +++ b/tools/fake-secure.conf @@ -1,2 +1 @@ -[database] -dburi=mysql+pymysql://nodepool@localhost/nodepool +# Empty diff --git a/tools/test-setup.sh b/tools/test-setup.sh deleted file mode 100755 index f4a045869..000000000 --- a/tools/test-setup.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -xe - -# This script will be run by OpenStack CI before unit tests are run, -# it sets up the test system as needed. -# Developers should setup their test systems in a similar way. - -# This setup needs to be run as a user that can run sudo. - -# The root password for the MySQL database; pass it in via -# MYSQL_ROOT_PW. -DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave} - -# This user and its password are used by the tests, if you change it, -# your tests might fail. -DB_USER=openstack_citest -DB_PW=openstack_citest - -sudo -H mysqladmin -u root password $DB_ROOT_PW - -# It's best practice to remove anonymous users from the database. If -# a anonymous user exists, then it matches first for connections and -# other connections from that host will not work. -sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " - DELETE FROM mysql.user WHERE User=''; - FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" - -# Now create our database. -mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " - SET default_storage_engine=MYISAM; - DROP DATABASE IF EXISTS openstack_citest; - CREATE DATABASE openstack_citest CHARACTER SET utf8;" diff --git a/tools/zuul-nodepool-integration/secure.conf b/tools/zuul-nodepool-integration/secure.conf index d29d9c094..b7db25411 100644 --- a/tools/zuul-nodepool-integration/secure.conf +++ b/tools/zuul-nodepool-integration/secure.conf @@ -1,2 +1 @@ -[database] -dburi=mysql+pymysql://nodepool@localhost/nodepool +# Empty