856a11e0e4
Note that this is the first part of the support. A second part involves dealing with the package names. Among the changes: - add several functions to determine some distro-specific behavior (how to call usermod, if some features are available on the distro, etc.) - correctly detect openSUSE and SLE in GetOSVersion, and set DISTRO accordingly - new is_suse() function to check if running on a SUSE-based distro - use zypper to install packages - adapt apache virtual host configuration for openSUSE - some simple fixes (path to pip, mysql service name) Change-Id: Id2f7c9e18a1c4a7b7cea262ea7959d183e4b0cf0
446 lines
17 KiB
Plaintext
446 lines
17 KiB
Plaintext
# lib/nova
|
||
# Functions to control the configuration and operation of the XXXX service
|
||
|
||
# Dependencies:
|
||
# ``functions`` file
|
||
# ``DEST``, ``DATA_DIR`` must be defined
|
||
# ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
|
||
# ``LIBVIRT_TYPE`` must be defined
|
||
# ``INSTANCE_NAME_PREFIX``, ``VOLUME_NAME_PREFIX`` must be defined
|
||
# ``KEYSTONE_TOKEN_FORMAT`` must be defined
|
||
|
||
# ``stack.sh`` calls the entry points in this order:
|
||
#
|
||
# install_nova
|
||
# configure_nova
|
||
# create_nova_conf
|
||
# init_nova
|
||
# start_nova
|
||
# stop_nova
|
||
# cleanup_nova
|
||
|
||
# Save trace setting
|
||
XTRACE=$(set +o | grep xtrace)
|
||
set +o xtrace
|
||
|
||
|
||
# Defaults
|
||
# --------
|
||
|
||
# Set up default directories
|
||
NOVA_DIR=$DEST/nova
|
||
NOVACLIENT_DIR=$DEST/python-novaclient
|
||
NOVA_STATE_PATH=${NOVA_STATE_PATH:=$DATA_DIR/nova}
|
||
# INSTANCES_PATH is the previous name for this
|
||
NOVA_INSTANCES_PATH=${NOVA_INSTANCES_PATH:=${INSTANCES_PATH:=$NOVA_STATE_PATH/instances}}
|
||
NOVA_AUTH_CACHE_DIR=${NOVA_AUTH_CACHE_DIR:-/var/cache/nova}
|
||
|
||
NOVA_CONF_DIR=/etc/nova
|
||
NOVA_CONF=$NOVA_CONF_DIR/nova.conf
|
||
NOVA_API_PASTE_INI=${NOVA_API_PASTE_INI:-$NOVA_CONF_DIR/api-paste.ini}
|
||
|
||
# Support entry points installation of console scripts
|
||
if [[ -d $NOVA_DIR/bin ]]; then
|
||
NOVA_BIN_DIR=$NOVA_DIR/bin
|
||
else
|
||
NOVA_BIN_DIR=/usr/local/bin
|
||
fi
|
||
|
||
# Set the paths of certain binaries
|
||
NOVA_ROOTWRAP=$(get_rootwrap_location nova)
|
||
|
||
# Allow rate limiting to be turned off for testing, like for Tempest
|
||
# NOTE: Set API_RATE_LIMIT="False" to turn OFF rate limiting
|
||
API_RATE_LIMIT=${API_RATE_LIMIT:-"True"}
|
||
|
||
# Nova supports pluggable schedulers. The default ``FilterScheduler``
|
||
# should work in most cases.
|
||
SCHEDULER=${SCHEDULER:-nova.scheduler.filter_scheduler.FilterScheduler}
|
||
|
||
QEMU_CONF=/etc/libvirt/qemu.conf
|
||
|
||
|
||
# Entry Points
|
||
# ------------
|
||
|
||
function add_nova_opt {
|
||
echo "$1" >>$NOVA_CONF
|
||
}
|
||
|
||
# Helper to clean iptables rules
|
||
function clean_iptables() {
|
||
# Delete rules
|
||
sudo iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "sudo iptables",$0}' | bash
|
||
# Delete nat rules
|
||
sudo iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-A" | sed "s/-A/-D/g" | awk '{print "sudo iptables -t nat",$0}' | bash
|
||
# Delete chains
|
||
sudo iptables -S -v | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "sudo iptables",$0}' | bash
|
||
# Delete nat chains
|
||
sudo iptables -S -v -t nat | sed "s/-c [0-9]* [0-9]* //g" | grep "nova" | grep "\-N" | sed "s/-N/-X/g" | awk '{print "sudo iptables -t nat",$0}' | bash
|
||
}
|
||
|
||
# cleanup_nova() - Remove residual data files, anything left over from previous
|
||
# runs that a clean run would need to clean up
|
||
function cleanup_nova() {
|
||
if is_service_enabled n-cpu; then
|
||
# Clean iptables from previous runs
|
||
clean_iptables
|
||
|
||
# Destroy old instances
|
||
instances=`sudo virsh list --all | grep $INSTANCE_NAME_PREFIX | sed "s/.*\($INSTANCE_NAME_PREFIX[0-9a-fA-F]*\).*/\1/g"`
|
||
if [ ! "$instances" = "" ]; then
|
||
echo $instances | xargs -n1 sudo virsh destroy || true
|
||
echo $instances | xargs -n1 sudo virsh undefine || true
|
||
fi
|
||
|
||
# Logout and delete iscsi sessions
|
||
sudo iscsiadm --mode node | grep $VOLUME_NAME_PREFIX | cut -d " " -f2 | xargs sudo iscsiadm --mode node --logout || true
|
||
sudo iscsiadm --mode node | grep $VOLUME_NAME_PREFIX | cut -d " " -f2 | sudo iscsiadm --mode node --op delete || true
|
||
|
||
# Clean out the instances directory.
|
||
sudo rm -rf $NOVA_INSTANCES_PATH/*
|
||
fi
|
||
}
|
||
|
||
# configure_novaclient() - Set config files, create data dirs, etc
|
||
function configure_novaclient() {
|
||
setup_develop $NOVACLIENT_DIR
|
||
}
|
||
|
||
# configure_nova_rootwrap() - configure Nova's rootwrap
|
||
function configure_nova_rootwrap() {
|
||
# Deploy new rootwrap filters files (owned by root).
|
||
# Wipe any existing rootwrap.d files first
|
||
if [[ -d $NOVA_CONF_DIR/rootwrap.d ]]; then
|
||
sudo rm -rf $NOVA_CONF_DIR/rootwrap.d
|
||
fi
|
||
# Deploy filters to /etc/nova/rootwrap.d
|
||
sudo mkdir -m 755 $NOVA_CONF_DIR/rootwrap.d
|
||
sudo cp $NOVA_DIR/etc/nova/rootwrap.d/*.filters $NOVA_CONF_DIR/rootwrap.d
|
||
sudo chown -R root:root $NOVA_CONF_DIR/rootwrap.d
|
||
sudo chmod 644 $NOVA_CONF_DIR/rootwrap.d/*
|
||
# Set up rootwrap.conf, pointing to /etc/nova/rootwrap.d
|
||
sudo cp $NOVA_DIR/etc/nova/rootwrap.conf $NOVA_CONF_DIR/
|
||
sudo sed -e "s:^filters_path=.*$:filters_path=$NOVA_CONF_DIR/rootwrap.d:" -i $NOVA_CONF_DIR/rootwrap.conf
|
||
sudo chown root:root $NOVA_CONF_DIR/rootwrap.conf
|
||
sudo chmod 0644 $NOVA_CONF_DIR/rootwrap.conf
|
||
# Specify rootwrap.conf as first parameter to nova-rootwrap
|
||
ROOTWRAP_SUDOER_CMD="$NOVA_ROOTWRAP $NOVA_CONF_DIR/rootwrap.conf *"
|
||
|
||
# Set up the rootwrap sudoers for nova
|
||
TEMPFILE=`mktemp`
|
||
echo "$USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD" >$TEMPFILE
|
||
chmod 0440 $TEMPFILE
|
||
sudo chown root:root $TEMPFILE
|
||
sudo mv $TEMPFILE /etc/sudoers.d/nova-rootwrap
|
||
}
|
||
|
||
# configure_nova() - Set config files, create data dirs, etc
|
||
function configure_nova() {
|
||
setup_develop $NOVA_DIR
|
||
|
||
# Put config files in ``/etc/nova`` for everyone to find
|
||
if [[ ! -d $NOVA_CONF_DIR ]]; then
|
||
sudo mkdir -p $NOVA_CONF_DIR
|
||
fi
|
||
sudo chown `whoami` $NOVA_CONF_DIR
|
||
|
||
cp -p $NOVA_DIR/etc/nova/policy.json $NOVA_CONF_DIR
|
||
|
||
configure_nova_rootwrap
|
||
|
||
if is_service_enabled n-api; then
|
||
# Use the sample http middleware configuration supplied in the
|
||
# Nova sources. This paste config adds the configuration required
|
||
# for Nova to validate Keystone tokens.
|
||
|
||
# Remove legacy paste config if present
|
||
rm -f $NOVA_DIR/bin/nova-api-paste.ini
|
||
|
||
# Get the sample configuration file in place
|
||
cp $NOVA_DIR/etc/nova/api-paste.ini $NOVA_CONF_DIR
|
||
|
||
# Rewrite the authtoken configuration for our Keystone service.
|
||
# This is a bit defensive to allow the sample file some variance.
|
||
sed -e "
|
||
/^admin_token/i admin_tenant_name = $SERVICE_TENANT_NAME
|
||
/admin_tenant_name/s/^.*$/admin_tenant_name = $SERVICE_TENANT_NAME/;
|
||
/admin_user/s/^.*$/admin_user = nova/;
|
||
/admin_password/s/^.*$/admin_password = $SERVICE_PASSWORD/;
|
||
s,%SERVICE_TENANT_NAME%,$SERVICE_TENANT_NAME,g;
|
||
s,%SERVICE_TOKEN%,$SERVICE_TOKEN,g;
|
||
" -i $NOVA_API_PASTE_INI
|
||
fi
|
||
|
||
if [[ "$KEYSTONE_TOKEN_FORMAT" == "PKI" ]]; then
|
||
iniset $NOVA_API_PASTE_INI filter:authtoken signing_dir $NOVA_AUTH_CACHE_DIR
|
||
fi
|
||
|
||
if is_service_enabled n-cpu; then
|
||
# Force IP forwarding on, just on case
|
||
sudo sysctl -w net.ipv4.ip_forward=1
|
||
|
||
# Attempt to load modules: network block device - used to manage qcow images
|
||
sudo modprobe nbd || true
|
||
|
||
# Check for kvm (hardware based virtualization). If unable to initialize
|
||
# kvm, we drop back to the slower emulation mode (qemu). Note: many systems
|
||
# come with hardware virtualization disabled in BIOS.
|
||
if [[ "$LIBVIRT_TYPE" == "kvm" ]]; then
|
||
sudo modprobe kvm || true
|
||
if [ ! -e /dev/kvm ]; then
|
||
echo "WARNING: Switching to QEMU"
|
||
LIBVIRT_TYPE=qemu
|
||
if which selinuxenabled 2>&1 > /dev/null && selinuxenabled; then
|
||
# https://bugzilla.redhat.com/show_bug.cgi?id=753589
|
||
sudo setsebool virt_use_execmem on
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Install and configure **LXC** if specified. LXC is another approach to
|
||
# splitting a system into many smaller parts. LXC uses cgroups and chroot
|
||
# to simulate multiple systems.
|
||
if [[ "$LIBVIRT_TYPE" == "lxc" ]]; then
|
||
if [[ "$os_PACKAGE" = "deb" ]]; then
|
||
if [[ ! "$DISTRO" > natty ]]; then
|
||
cgline="none /cgroup cgroup cpuacct,memory,devices,cpu,freezer,blkio 0 0"
|
||
sudo mkdir -p /cgroup
|
||
if ! grep -q cgroup /etc/fstab; then
|
||
echo "$cgline" | sudo tee -a /etc/fstab
|
||
fi
|
||
if ! mount -n | grep -q cgroup; then
|
||
sudo mount /cgroup
|
||
fi
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if is_service_enabled quantum && is_quantum_ovs_base_plugin "$Q_PLUGIN" && ! sudo grep -q '^cgroup_device_acl' $QEMU_CONF ; then
|
||
# Add /dev/net/tun to cgroup_device_acls, needed for type=ethernet interfaces
|
||
cat <<EOF | sudo tee -a $QEMU_CONF
|
||
cgroup_device_acl = [
|
||
"/dev/null", "/dev/full", "/dev/zero",
|
||
"/dev/random", "/dev/urandom",
|
||
"/dev/ptmx", "/dev/kvm", "/dev/kqemu",
|
||
"/dev/rtc", "/dev/hpet","/dev/net/tun",
|
||
]
|
||
EOF
|
||
fi
|
||
|
||
if [[ "$os_PACKAGE" = "deb" ]]; then
|
||
LIBVIRT_DAEMON=libvirt-bin
|
||
else
|
||
# http://wiki.libvirt.org/page/SSHPolicyKitSetup
|
||
if ! grep ^libvirtd: /etc/group >/dev/null; then
|
||
sudo groupadd libvirtd
|
||
fi
|
||
sudo bash -c 'cat <<EOF >/etc/polkit-1/localauthority/50-local.d/50-libvirt-remote-access.pkla
|
||
[libvirt Management Access]
|
||
Identity=unix-group:libvirtd
|
||
Action=org.libvirt.unix.manage
|
||
ResultAny=yes
|
||
ResultInactive=yes
|
||
ResultActive=yes
|
||
EOF'
|
||
LIBVIRT_DAEMON=libvirtd
|
||
fi
|
||
|
||
# The user that nova runs as needs to be member of **libvirtd** group otherwise
|
||
# nova-compute will be unable to use libvirt.
|
||
add_user_to_group `whoami` libvirtd
|
||
|
||
# libvirt detects various settings on startup, as we potentially changed
|
||
# the system configuration (modules, filesystems), we need to restart
|
||
# libvirt to detect those changes.
|
||
restart_service $LIBVIRT_DAEMON
|
||
|
||
|
||
# Instance Storage
|
||
# ----------------
|
||
|
||
# Nova stores each instance in its own directory.
|
||
mkdir -p $NOVA_INSTANCES_PATH
|
||
|
||
# You can specify a different disk to be mounted and used for backing the
|
||
# virtual machines. If there is a partition labeled nova-instances we
|
||
# mount it (ext filesystems can be labeled via e2label).
|
||
if [ -L /dev/disk/by-label/nova-instances ]; then
|
||
if ! mount -n | grep -q $NOVA_INSTANCES_PATH; then
|
||
sudo mount -L nova-instances $NOVA_INSTANCES_PATH
|
||
sudo chown -R `whoami` $NOVA_INSTANCES_PATH
|
||
fi
|
||
fi
|
||
|
||
# Clean up old instances
|
||
cleanup_nova
|
||
fi
|
||
}
|
||
|
||
# create_nova_conf() - Create a new nova.conf file
|
||
function create_nova_conf() {
|
||
# Remove legacy ``nova.conf``
|
||
rm -f $NOVA_DIR/bin/nova.conf
|
||
|
||
# (Re)create ``nova.conf``
|
||
rm -f $NOVA_CONF
|
||
add_nova_opt "[DEFAULT]"
|
||
add_nova_opt "verbose=True"
|
||
add_nova_opt "auth_strategy=keystone"
|
||
add_nova_opt "allow_resize_to_same_host=True"
|
||
add_nova_opt "api_paste_config=$NOVA_API_PASTE_INI"
|
||
add_nova_opt "rootwrap_config=$NOVA_CONF_DIR/rootwrap.conf"
|
||
add_nova_opt "compute_scheduler_driver=$SCHEDULER"
|
||
add_nova_opt "dhcpbridge_flagfile=$NOVA_CONF"
|
||
add_nova_opt "force_dhcp_release=True"
|
||
add_nova_opt "fixed_range=$FIXED_RANGE"
|
||
add_nova_opt "s3_host=$SERVICE_HOST"
|
||
add_nova_opt "s3_port=$S3_SERVICE_PORT"
|
||
add_nova_opt "osapi_compute_extension=nova.api.openstack.compute.contrib.standard_extensions"
|
||
add_nova_opt "my_ip=$HOST_IP"
|
||
local dburl
|
||
database_connection_url dburl nova
|
||
add_nova_opt "sql_connection=$dburl"
|
||
add_nova_opt "libvirt_type=$LIBVIRT_TYPE"
|
||
add_nova_opt "libvirt_cpu_mode=none"
|
||
add_nova_opt "instance_name_template=${INSTANCE_NAME_PREFIX}%08x"
|
||
|
||
if is_service_enabled n-api; then
|
||
add_nova_opt "enabled_apis=$NOVA_ENABLED_APIS"
|
||
fi
|
||
if is_service_enabled cinder; then
|
||
add_nova_opt "volume_api_class=nova.volume.cinder.API"
|
||
fi
|
||
if [ -n "$NOVA_STATE_PATH" ]; then
|
||
add_nova_opt "state_path=$NOVA_STATE_PATH"
|
||
add_nova_opt "lock_path=$NOVA_STATE_PATH"
|
||
fi
|
||
if [ -n "$NOVA_INSTANCES_PATH" ]; then
|
||
add_nova_opt "instances_path=$NOVA_INSTANCES_PATH"
|
||
fi
|
||
if [ "$MULTI_HOST" != "False" ]; then
|
||
add_nova_opt "multi_host=True"
|
||
add_nova_opt "send_arp_for_ha=True"
|
||
fi
|
||
if [ "$SYSLOG" != "False" ]; then
|
||
add_nova_opt "use_syslog=True"
|
||
fi
|
||
if [ "$API_RATE_LIMIT" != "True" ]; then
|
||
add_nova_opt "api_rate_limit=False"
|
||
fi
|
||
if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
|
||
# Add color to logging output
|
||
add_nova_opt "logging_context_format_string=%(asctime)s %(color)s%(levelname)s %(name)s [[01;36m%(request_id)s [00;36m%(user_name)s %(project_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
|
||
add_nova_opt "logging_default_format_string=%(asctime)s %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
|
||
add_nova_opt "logging_debug_format_suffix=[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
|
||
add_nova_opt "logging_exception_prefix=%(color)s%(asctime)s TRACE %(name)s [01;35m%(instance)s[00m"
|
||
else
|
||
# Show user_name and project_name instead of user_id and project_id
|
||
add_nova_opt "logging_context_format_string=%(asctime)s %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
|
||
fi
|
||
if is_service_enabled ceilometer; then
|
||
add_nova_opt "instance_usage_audit=True"
|
||
add_nova_opt "instance_usage_audit_period=hour"
|
||
add_nova_opt "notification_driver=nova.openstack.common.notifier.rabbit_notifier"
|
||
add_nova_opt "notification_driver=ceilometer.compute.nova_notifier"
|
||
fi
|
||
|
||
|
||
# Provide some transition from ``EXTRA_FLAGS`` to ``EXTRA_OPTS``
|
||
if [[ -z "$EXTRA_OPTS" && -n "$EXTRA_FLAGS" ]]; then
|
||
EXTRA_OPTS=$EXTRA_FLAGS
|
||
fi
|
||
|
||
# Define extra nova conf flags by defining the array ``EXTRA_OPTS``.
|
||
# For Example: ``EXTRA_OPTS=(foo=true bar=2)``
|
||
for I in "${EXTRA_OPTS[@]}"; do
|
||
# Attempt to convert flags to options
|
||
add_nova_opt ${I//--}
|
||
done
|
||
}
|
||
|
||
# init_nova() - Initialize databases, etc.
|
||
function init_nova() {
|
||
# Nova Database
|
||
# -------------
|
||
|
||
# All nova components talk to a central database. We will need to do this step
|
||
# only once for an entire cluster.
|
||
|
||
if is_service_enabled $DATABASE_BACKENDS && is_service_enabled nova; then
|
||
# (Re)create nova database
|
||
# Explicitly use latin1: to avoid lp#829209, nova expects the database to
|
||
# use latin1 by default, and then upgrades the database to utf8 (see the
|
||
# 082_essex.py in nova)
|
||
recreate_database nova latin1
|
||
|
||
# (Re)create nova database
|
||
$NOVA_BIN_DIR/nova-manage db sync
|
||
fi
|
||
|
||
if [[ "$KEYSTONE_TOKEN_FORMAT" == "PKI" ]]; then
|
||
# Create cache dir
|
||
sudo mkdir -p $NOVA_AUTH_CACHE_DIR
|
||
sudo chown `whoami` $NOVA_AUTH_CACHE_DIR
|
||
fi
|
||
}
|
||
|
||
# install_novaclient() - Collect source and prepare
|
||
function install_novaclient() {
|
||
git_clone $NOVACLIENT_REPO $NOVACLIENT_DIR $NOVACLIENT_BRANCH
|
||
}
|
||
|
||
# install_nova() - Collect source and prepare
|
||
function install_nova() {
|
||
if is_service_enabled n-cpu; then
|
||
if [[ "$os_PACKAGE" = "deb" ]]; then
|
||
LIBVIRT_PKG_NAME=libvirt-bin
|
||
else
|
||
LIBVIRT_PKG_NAME=libvirt
|
||
fi
|
||
install_package $LIBVIRT_PKG_NAME
|
||
# Install and configure **LXC** if specified. LXC is another approach to
|
||
# splitting a system into many smaller parts. LXC uses cgroups and chroot
|
||
# to simulate multiple systems.
|
||
if [[ "$LIBVIRT_TYPE" == "lxc" ]]; then
|
||
if [[ "$os_PACKAGE" = "deb" ]]; then
|
||
if [[ "$DISTRO" > natty ]]; then
|
||
install_package cgroup-lite
|
||
fi
|
||
else
|
||
### FIXME(dtroyer): figure this out
|
||
echo "RPM-based cgroup not implemented yet"
|
||
yum_install libcgroup-tools
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
git_clone $NOVA_REPO $NOVA_DIR $NOVA_BRANCH
|
||
}
|
||
|
||
# start_nova() - Start running processes, including screen
|
||
function start_nova() {
|
||
# The group **libvirtd** is added to the current user in this script.
|
||
# Use 'sg' to execute nova-compute as a member of the **libvirtd** group.
|
||
# ``screen_it`` checks ``is_service_enabled``, it is not needed here
|
||
screen_it n-cpu "cd $NOVA_DIR && sg libvirtd $NOVA_BIN_DIR/nova-compute"
|
||
screen_it n-crt "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-cert"
|
||
screen_it n-net "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-network"
|
||
screen_it n-sch "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-scheduler"
|
||
screen_it n-novnc "cd $NOVNC_DIR && ./utils/nova-novncproxy --config-file $NOVA_CONF --web ."
|
||
screen_it n-xvnc "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-xvpvncproxy --config-file $NOVA_CONF"
|
||
screen_it n-cauth "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-consoleauth"
|
||
screen_it n-cond "cd $NOVA_DIR && $NOVA_BIN_DIR/nova-conductor"
|
||
}
|
||
|
||
# stop_nova() - Stop running processes (non-screen)
|
||
function stop_nova() {
|
||
# Kill the nova screen windows
|
||
for serv in n-api n-cpu n-crt n-net n-sch n-novnc n-xvnc n-cauth n-cond; do
|
||
screen -S $SCREEN_NAME -p $serv -X kill
|
||
done
|
||
}
|
||
|
||
# Restore xtrace
|
||
$XTRACE
|