Collect filesystem initial commit

This tool is used for storing large collect files when reporting bugs
via Launchpad. Simply use the tool to upload a file and then add its URL
location into the Launchpad bug.  You no longer need to split up large
collect files and attach multiple files to your bugs.
This tool is designed using Flask Web Framework, meanwhile docker is
used for easy deployment.
The directory /app holds the Flask application, /db contains the script
for database initialization. Bash file start.sh includes the commands
used for the first time docker deployment.
The file update.sh let developers push their changes to the docker
repository. Change the repository's name and tag accordingly, in this
case it is nchen1windriver/collect and demo.
A config file was added for security purpose in this commit.

Change-Id: I192c3fca541f99773e0395418a9f11e01c27a5a7
Signed-off-by: Nathan Chen <nathan.chen@windriver.com>

.gitignore

@@ -1,3 +1,4 @@
 .tox
 *.egg-info
 *.swp
+.idea

tools/collect_filesystem/Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y && \
+    apt-get install -y python3-dev python3-pip libffi-dev
+
+RUN pip3 install --upgrade pip
+
+RUN pip3 install python-magic flask_mail flask_openid werkzeug pymysql launchpadlib apscheduler
+
+WORKDIR /app
+
+COPY app/ /app/
+
+RUN pip3 install -r requirements.txt
+
+EXPOSE 5000:5000
+
+CMD ["python3", "app.py"]

tools/collect_filesystem/app/app.py

@@ -0,0 +1,842 @@
+import os
+import shutil
+import logging
+import zipfile
+from datetime import datetime
+from datetime import timedelta
+from functools import wraps
+import tarfile
+from mail_config import custom_mail_password
+from mail_config import custom_mail_server
+from mail_config import custom_mail_username
+from mail_config import custom_server_admins
+
+import magic
+from flask_mail import Mail
+from flask_mail import Message
+from urllib.parse import quote
+from urllib.parse import unquote
+
+from flask import Flask
+from flask import flash
+from flask import request
+from flask import redirect
+from flask import render_template
+from flask import g
+from flask import session
+from flask import url_for
+from flask import abort
+from flask import send_file
+from flask import make_response
+from flask import jsonify
+from flask import after_this_request
+import flask_openid
+from openid.extensions import pape
+from werkzeug.exceptions import RequestEntityTooLarge
+from werkzeug.utils import secure_filename
+import pymysql.cursors
+from launchpadlib.launchpad import Launchpad
+import atexit
+
+from apscheduler.schedulers.background import BackgroundScheduler
+
+logging.basicConfig(filename='collect.log', level=logging.INFO, format='%(asctime)s:%(levelname)s:%(message)s')
+
+app = Flask(__name__)
+app.config['BASE_DIR'] = os.path.abspath(
+    os.path.dirname(__file__)
+)
+app.config['MAX_CONTENT_LENGTH'] = 10 * 1024 * 1024 * 1000
+app.testing = False
+app.config.update(
+    # Gmail sender settings
+    MAIL_SERVER=custom_mail_server,
+    MAIL_PORT=465,
+    MAIL_USE_TLS=False,
+    MAIL_USE_SSL=True,
+    MAIL_USERNAME=custom_mail_username,
+    MAIL_PASSWORD=custom_mail_password,
+    MAIL_DEFAULT_SENDER=custom_mail_username,
+    SECRET_KEY='development key',
+    DEBUG=True
+)
+
+mail = Mail(app)
+
+ALLOWED_EXTENSIONS = set(['log', 'txt', 'zip', 'tar', 'tgz'])
+DISALLOWED_EXTENSIONS = set(['exe', 'mp4', 'avi', 'mkv'])
+ALLOWED_MIME_TYPES = set(['text/plain', 'application/x-bzip2', 'application/zip', 'application/x-gzip',
+                          'application/x-tar'])
+DISALLOWED_MIME_TYPES = set(['application/x-dosexec', 'application/x-msdownload'])
+TARGETED_TAR_CONTENTS = set(['controller', 'storage', 'compute'])
+SERVER_ADMINS = custom_server_admins
+THRESHOLD = 0.8
+
+oid = flask_openid.OpenID(app, safe_roots=[], extension_responses=[pape.Response])
+
+
+def delete_old_files():
+    time_before = datetime.now() - timedelta(months=6)
+    with connect().cursor() as cursor:
+        files_sql = "SELECT name, user_id, launchpad_id FROM files WHERE modified_date<%s;"
+        cursor.execute(files_sql, (time_before,))
+        files = cursor.fetchall()
+        for file in files:
+            os.remove(os.path.join(app.config['BASE_DIR'], 'files', file['user_id'], str(file['launchpad_id']),
+                                   file['name']))
+            logging.info('Outdated file deleted: {}/{}/{}'.format(file['user_id'], file['launchpad_id'], file['name']))
+
+        sql = "DELETE FROM files WHERE modified_date<%s;"
+        cursor.execute(sql, (time_before,))
+        cursor.close()
+
+
+def check_launchpads():
+    with connect().cursor() as cursor:
+        launchpads_sql = "SELECT * FROM launchpads"
+        cursor.execute(launchpads_sql)
+        launchpads = cursor.fetchall()
+        for launchpad in launchpads:
+            launchpad_info = get_launchpad_info(launchpad['id'])
+            if launchpad_info[1] is True:
+                sql = "DELETE FROM launchpads WHERE id=%s;"
+                cursor.execute(sql, (launchpad['id'],))
+            elif launchpad_info[0] != launchpad['id']:
+                sql = "UPDATE launchpads SET title = %s WHERE id = %s;"
+                cursor.execute(sql, (launchpad_info[0], launchpad['id'],))
+        cursor.close()
+
+
+def free_storage():
+    with connect().cursor() as cursor:
+        files_sql = "SELECT id, name, user_id, launchpad_id FROM files ORDER BY modified_date;"
+        cursor.execute(files_sql)
+        files = cursor.fetchall()
+        for file in files:
+            os.remove(os.path.join(app.config['BASE_DIR'], 'files', str(file['user_id']), str(file['launchpad_id']),
+                                   file['name']))
+            logging.info('Outdated file deleted: {}/{}/{}'.format(file['user_id'], file['launchpad_id'], file['name']))
+            sql = "DELETE FROM files WHERE id=%s;"
+            cursor.execute(sql, (file['id'],))
+            if get_usage_info()[0] < THRESHOLD:
+                break
+        cursor.close()
+
+
+def send_weekly_reports():
+    usage_info = get_usage_info()
+    subject = 'Weekly Report'
+    rounded_usage = round(usage_info[0], 4)
+    free_space = round(usage_info[1]/1000000, 2)
+    with connect().cursor() as cursor:
+        sql = 'SELECT n.name, i.upload_count, i.total_size FROM openid_users n RIGHT JOIN ' \
+              '(SELECT user_id, COUNT(*) AS upload_count, ROUND(SUM(file_size)/1000000, 2) AS total_size FROM files ' \
+              'WHERE modified_date > NOW() - INTERVAL 1 WEEK GROUP BY user_id)i ' \
+              'ON n.id = i.user_id;'
+        cursor.execute(sql)
+        reports_by_user = cursor.fetchall()
+    reports_by_user_table = '<table><tr>' \
+                            '<th>Name</th>' \
+                            '<th>Number of Uploads</th>' \
+                            '<th>Total Upload Size</th>' \
+                            '</tr>'
+    for report in reports_by_user:
+        reports_by_user_table = reports_by_user_table + '<tr>' \
+                                                        '<td>%s</td><td>%s</td><td>%s</td>' \
+                                                        '</tr>' % (report["name"], report["upload_count"], report["total_size"])
+    reports_by_user_table = reports_by_user_table + '</tr></table>'
+    with app.app_context():
+        for admin in SERVER_ADMINS:
+            html_body = '<p>Hello %s,' \
+                        '<br>Here is the upload report for last week:<br>%s' \
+                        '<br>There are %s space used and %s MB of space left.' \
+                        % (admin["name"], reports_by_user_table, rounded_usage, usage_info[0])
+
+            msg = Message(subject,
+                          html=html_body,
+                          recipients=[admin['email']])
+            mail.send(msg)
+
+
+# @app.route('/storage_full/')
+def if_storage_full():
+    usage_info = get_usage_info()
+    if usage_info[0] > THRESHOLD:
+        subject = 'Storage is Nearly Full'
+        rounded_usage = round(usage_info[0], 4)
+        with app.app_context():
+            for admin in SERVER_ADMINS:
+                html_body = '<p>Hello %s,' \
+                            '<br>The logfiles uploaded took %s of the server\'s total disk space.' \
+                            'Oldest files will be deleted to free space.</p>' % (admin["name"], rounded_usage)
+
+                msg = Message(subject,
+                              html=html_body,
+                              recipients=[admin['email']])
+                mail.send(msg)
+        free_storage()
+
+        # with smtplib.SMTP('mail.wrs.com', 587) as smtp:
+        #     # smtp.ehlo()
+        #     smtp.starttls()
+        #     smtp.ehlo()
+        #
+        #     # smtp.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
+        #     smtp.login('Nathan.Chen@windriver.com', 'MyPassword')
+        #
+        #     subject = 'Storage is full'
+        #     body = 'Do something'
+        #
+        #     msg = f'Subject: {subject}\n\n{body}'
+        #
+        #     smtp.sendmail('Nathan.Chen@windriver.com', 'wrcollecttesting1@gmail.com', msg)
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(func=delete_old_files, trigger="cron", day='1')
+scheduler.add_job(func=check_launchpads, trigger="cron", day_of_week='mon-fri')
+scheduler.add_job(func=if_storage_full, trigger="cron", minute='00')
+scheduler.add_job(func=send_weekly_reports, trigger="cron", day_of_week='mon')
+scheduler.start()
+
+atexit.register(lambda: scheduler.shutdown())
+
+
+# @app.route('/usage/')
+def get_usage_info():
+    # return str(shutil.disk_usage(app.config['BASE_DIR']))
+    # return str(shutil.disk_usage(os.path.join(app.config['BASE_DIR'], 'files')).used)
+    # return str(shutil.disk_usage(os.path.join(app.config['BASE_DIR'], 'files')))
+    disk_usage_info = shutil.disk_usage(os.path.join(app.config['BASE_DIR'], 'files'))
+    usage_perc = disk_usage_info.used/disk_usage_info.total
+    return [usage_perc, disk_usage_info.free]
+    # return str(usage_perc)
+
+
+def is_allowed(filename):
+    return '.' in filename and (filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
+                                or filename.rsplit('.', 2)[1] == 'tar') \
+           and filename.rsplit('.', 1)[1] not in DISALLOWED_EXTENSIONS
+
+
+def get_size(fobj):
+    if fobj.content_length:
+        return fobj.content_length
+
+    try:
+        pos = fobj.tell()
+        fobj.seek(0, 2)  # seek to end
+        size = fobj.tell()
+        fobj.seek(pos)  # back to original position
+        return size
+    except (AttributeError, IOError):
+        pass
+
+    # in-memory file object that doesn't support seeking or tell
+    return 0  # assume small enough
+
+
+def connect():
+    return pymysql.connect(host='db',
+                           user='root',
+                           password='Wind2019',
+                           db='collect',
+                           charset='utf8mb4',
+                           cursorclass=pymysql.cursors.DictCursor,
+                           autocommit=True)
+
+
+def confirmation_required(desc_fn):
+    def inner(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            if request.args.get('confirm') != '1':
+                desc = desc_fn()
+                return redirect(url_for('confirm', desc=desc, action_url=quote(request.url)))
+            return f(*args, **kwargs)
+
+        return wrapper
+
+    return inner
+
+
+def get_launchpad_info(lp):
+    lp_id = lp
+    if isinstance(lp, dict):
+        lp_id = int(lp['launchpad_id'])
+    cachedir = os.path.join(app.config['BASE_DIR'], '.launchpadlib/cache/')
+    launchpad = Launchpad.login_anonymously('just testing', 'production', cachedir, version='devel')
+    try:
+        bug_one = launchpad.bugs[lp_id]
+        # return str(type(bug_one))
+        # return bug_one.bug_tasks.entries[0]['title']
+        is_starlingx = any(entry['bug_target_name'] == 'starlingx' for entry in bug_one.bug_tasks.entries)
+        if not is_starlingx:
+            return 'You need to choose a valid StarlingX Launchpad'
+        closed = all(entry['date_closed'] is not None for entry in bug_one.bug_tasks.entries)
+        return [bug_one.title, closed]
+    except KeyError:
+        return 'Launchpad bug id does not exist'
+
+
+@app.errorhandler(413)
+@app.errorhandler(RequestEntityTooLarge)
+def error413(e):
+    # flash(u'Error: File size exceeds the 16GB limit')
+    # return render_template('index.html'), 413
+    # return render_template('413.html'), 413
+    return 'File Too Large', 413
+
+
+@app.errorhandler(401)
+def error401(e):
+    flash(u'Error: You are not logged in')
+    return redirect(url_for('login', next=oid.get_next_url()))
+
+
+@app.route('/confirm')
+def confirm():
+    desc = request.args['desc']
+    action_url = unquote(request.args['action_url'])
+
+    return render_template('_confirm.html', desc=desc, action_url=action_url)
+
+
+def confirm_delete_profile():
+    return "Are you sure you want to delete your profile? All your files will be removed."
+
+
+def confirm_delete_file():
+    return "Are you sure you want to delete this file?"
+
+
+@app.before_request
+def before_request():
+    g.user = None
+    g.connection = connect()
+    if 'openid' in session:
+        with g.connection.cursor() as cursor:
+            sql = "select * from openid_users where openid = %s;"
+            cursor.execute(sql, (session['openid'],))
+            g.user = cursor.fetchone()
+            cursor.close()
+
+
+@app.after_request
+def after_request(response):
+    return response
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@app.route('/login/', methods=['GET', 'POST'])
+@oid.loginhandler
+def login():
+    if g.user is not None:
+        return redirect(oid.get_next_url())
+    if request.method == 'POST':
+        openid = "https://launchpad.net/~" + request.form.get('openid')
+        if openid:
+            pape_req = pape.Request([])
+            return oid.try_login(openid, ask_for=['email', 'nickname'],
+                                 ask_for_optional=['fullname'],
+                                 extensions=[pape_req])
+    return render_template('login.html', next=oid.get_next_url(),
+                           error=oid.fetch_error())
+
+
+@oid.after_login
+def create_or_login(resp):
+    """This is called when login with OpenID succeeded and it's not
+    necessary to figure out if this is the users's first login or not.
+    This function has to redirect otherwise the user will be presented
+    with a terrible URL which we certainly don't want.
+    """
+    session['openid'] = resp.identity_url
+    if 'pape' in resp.extensions:
+        pape_resp = resp.extensions['pape']
+        session['auth_time'] = pape_resp.auth_time
+    with g.connection.cursor() as cursor:
+        sql = "select * from openid_users where openid = %s;"
+        cursor.execute(sql, (resp.identity_url,))
+        user = cursor.fetchone()
+        cursor.close()
+    # user = User.query.filter_by(openid=resp.identity_url).first()
+    if user is not None:
+        flash(u'Successfully signed in')
+        g.user = user
+        return redirect(oid.get_next_url())
+    return redirect(url_for('create_profile', next=oid.get_next_url(),
+                            name=resp.fullname or resp.nickname,
+                            email=resp.email))
+
+
+@app.route('/create-profile/', methods=['GET', 'POST'])
+def create_profile():
+    """If this is the user's first login, the create_or_login function
+    will redirect here so that the user can set up his profile.
+    """
+    if g.user is not None or 'openid' not in session:
+        return redirect(url_for('index'))
+    if request.method == 'POST':
+        name = request.form['name']
+        email = request.form['email']
+        if not name:
+            flash(u'Error: you have to provide a name')
+        elif '@' not in email:
+            flash(u'Error: you have to enter a valid email address')
+        else:
+            flash(u'Profile successfully created')
+            with g.connection.cursor() as cursor:
+                sql = "INSERT INTO openid_users (name, email, openid) VALUES (%s, %s, %s);"
+                cursor.execute(sql, (name, email, session['openid'],))
+                sql = "SELECT id FROM openid_users WHERE openid = %s;"
+                cursor.execute(sql, (session['openid'],))
+                user_id = cursor.fetchone()['id']
+                _dir = os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(user_id))
+                os.mkdir(_dir)
+                cursor.close()
+                return redirect(oid.get_next_url())
+    return render_template('create_profile.html', next_url=oid.get_next_url())
+
+
+@app.route('/profile/', methods=['GET', 'POST'])
+def edit_profile():
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    # form = dict(name=g.user.name, email=g.user.email)
+    # g_user_name = g.user.name
+    # g_user_email = g.user.email
+    form = {'name': g.user['name'], 'email': g.user['email']}
+    # form = {'name': 1, 'email': 1}
+    # form['name'] = g.user.name
+    # form['email'] = g.user.email
+    if request.method == 'POST':
+        if 'delete' in request.form:
+            with g.connection.cursor() as cursor:
+                sql = "DELETE FROM openid_users WHERE openid=%s;"
+                cursor.execute(sql, (session['openid'],))
+                cursor.close()
+                shutil.rmtree(os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(g.user['id'])))
+                return redirect(oid.get_next_url())
+            session['openid'] = None
+            flash(u'Profile deleted')
+            return redirect(url_for('index'))
+        form['name'] = request.form['name']
+        form['email'] = request.form['email']
+        if not form['name']:
+            flash(u'Error: you have to provide a name')
+        elif '@' not in form['email']:
+            flash(u'Error: you have to enter a valid email address')
+        else:
+            flash(u'Profile successfully created')
+            g.user['name'] = form['name']
+            g.user['email'] = form['email']
+            with g.connection.cursor() as cursor:
+                sql = "UPDATE openid_users SET name = %s, email = %s WHERE id = %s;"
+                cursor.execute(sql, (g.user['name'], g.user['email'], g.user['id'],))
+                cursor.close()
+                return redirect(oid.get_next_url())
+            # db_session.commit()
+            # return redirect(url_for('edit_profile'))
+    return render_template('edit_profile.html', form=form)
+
+
+@app.route('/logout/')
+def logout():
+    session.pop('openid', None)
+    flash(u'You have been signed out')
+    return redirect(oid.get_next_url())
+
+
+@app.route('/check_launchpad/<launchpad_id>', methods=['GET', 'POST'])
+def check_launchpad(launchpad_id):
+    with g.connection.cursor() as cursor:
+        sql = "SELECT * FROM launchpads WHERE id = %s;"
+        cursor.execute(sql, (launchpad_id,))
+        launchpad_info = cursor.fetchone()
+        if launchpad_info:
+            launchpad_title = launchpad_info["title"]
+        else:
+            try:
+                launchpad_info = get_launchpad_info(int(launchpad_id))
+            except ValueError:
+                res = make_response("Error: Launchpad bug id does not exist", 400)
+                return res
+            if launchpad_info == 'Launchpad bug id does not exist':
+                res = make_response("Error: Launchpad bug id does not exist", 400)
+                return res
+            elif launchpad_info == 'You need to choose a valid StarlingX Launchpad':
+                res = make_response("Error: You need to choose a valid StarlingX Launchpad", 400)
+                return res
+            elif launchpad_info[1] is True:
+                res = make_response("Error: Launchpad bug is closed", 400)
+                return res
+            sql = "INSERT INTO launchpads (id, title) VALUES (%s, %s);"
+            cursor.execute(sql, (launchpad_id, launchpad_info[0],))
+            launchpad_title = launchpad_info[0]
+        res = make_response(launchpad_title, 200)
+        return res
+
+
+@app.route('/upload/', methods=['GET', 'POST'])
+def upload():
+    try:
+        if g.user is None:
+            abort(401)
+        if request.method == 'POST':
+            launchpad_id = request.args.get('launchpad_id')
+            if launchpad_id == '':
+                res = make_response(jsonify({"message": "Error: you did not supply a valid Launchpad ID"}), 400)
+                return res
+            launchpad_info = 0
+
+            # with g.connection.cursor() as cursor:
+            #     sql = "SELECT id FROM launchpads WHERE id = %s;"
+            #     cursor.execute(sql, (launchpad_id,))
+            #     if not cursor.fetchone():
+            #         try:
+            #             launchpad_info = get_launchpad_info(int(launchpad_id))
+            #         except ValueError:
+            #             res = make_response(jsonify({"message": "Error: Launchpad bug id does not exist"}), 400)
+            #             return res
+            #         if launchpad_info == 'Launchpad bug id does not exist' or launchpad_info[1] is True:
+            #             res = make_response(jsonify({"message": "Error: Launchpad bug id does not exist"}), 400)
+            #             return res
+            #         sql = "INSERT INTO launchpads (id, title) VALUES (%s, %s);"
+            #         cursor.execute(sql, (launchpad_id, launchpad_info[0],))
+
+            # file_list = request.files.getlist('file')
+            # file_list = request.files['file']
+            # file_list = request.files.get('file[]')
+            #
+            # res = make_response(jsonify({"message": str(file_list)}), 200)
+            # return res
+
+            # file_size = get_size(f)
+            # if file_size > MAX_CONTENT_LENGTH:
+            #     flash(u'Error: File size exceeds the 10GB limit')
+            #     return redirect(oid.get_next_url())
+            # for f in file_list:
+            f = request.files['file']
+            if f and is_allowed(f.filename):
+                _launchpad_dir = os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(g.user['id']), launchpad_id)
+                conflict = request.args.get('conflict')
+                final_filename = secure_filename(f.filename)
+                if conflict == '1':
+                    if final_filename.rsplit('.', 2)[1] == 'tar' and len(final_filename.rsplit('.', 2)) == 3:
+                        tail = 1
+                        while True:
+                            new_filename = final_filename.rsplit('.', 2)[0] + "_" + str(tail) + '.' \
+                                + final_filename.rsplit('.', 2)[1] + '.' + final_filename.rsplit('.', 2)[2]
+                            with g.connection.cursor() as cursor:
+                                file_name_sql = "SELECT id FROM files WHERE launchpad_id=%s AND name=%s AND user_id=%s;"
+                                cursor.execute(file_name_sql, (launchpad_id, new_filename, g.user['id']))
+                                if cursor.fetchone():
+                                    tail = tail + 1
+                                else:
+                                    break
+                    else:
+                        tail = 1
+                        while True:
+                            new_filename = final_filename.rsplit('.', 1)[0] + "_" + str(tail) + '.' \
+                                + final_filename.rsplit('.', 1)[1]
+                            with g.connection.cursor() as cursor:
+                                file_name_sql = "SELECT id FROM files WHERE launchpad_id=%s AND name=%s AND user_id=%s;"
+                                cursor.execute(file_name_sql, (launchpad_id, new_filename, g.user['id']))
+                                if cursor.fetchone():
+                                    tail = tail + 1
+                                else:
+                                    break
+                    final_filename = new_filename
+                if not os.path.isdir(_launchpad_dir):
+                    os.mkdir(_launchpad_dir)
+                _full_dir = os.path.join(_launchpad_dir, final_filename)
+                f.save(_full_dir)
+                file_size = os.stat(_full_dir).st_size
+                # mimetype = f.mimetype
+                # flash(mimetype)
+                mime = magic.Magic(mime=True)
+                mimetype = mime.from_file(_full_dir)
+                if mimetype.startswith('image/', 0) or mimetype.startswith('video/', 0)\
+                        or mimetype in DISALLOWED_MIME_TYPES:
+                    os.remove(_full_dir)
+                    res = make_response(jsonify({
+                        "message": "Error: you did not supply a valid file in your request"}), 400)
+                    return res
+                if mimetype in ['application/x-gzip', 'application/x-tar']:
+                    tar = tarfile.open(_full_dir)
+                    if not any((any(x in y for x in TARGETED_TAR_CONTENTS) for y in tar.getnames())):
+                        res = make_response(jsonify({
+                            "message": "Error: you did not supply a valid collect file in your request"}), 400)
+                        logging.info(tar.getnames())
+                        return res
+                if conflict == '0':
+                    with g.connection.cursor() as cursor:
+                        sql = "UPDATE files SET modified_date = %s, file_size = %s " \
+                              "WHERE user_id = %s AND name = %s AND launchpad_id = %s;"
+                        cursor.execute(
+                            sql, (datetime.now(), file_size, g.user['id'], final_filename, launchpad_id))
+                        cursor.close()
+                    logging.info('User#{} re-uploaded file {} under launchpad bug#{}'.
+                                 format(g.user['id'], final_filename, launchpad_id))
+                else:
+                    with g.connection.cursor() as cursor:
+                        sql = "INSERT INTO files (name, user_id, launchpad_id, modified_date, file_size)" \
+                              "VALUES (%s, %s, %s, %s, %s);"
+                        cursor.execute(sql, (final_filename, g.user['id'], launchpad_id, datetime.now(),
+                                             file_size,))
+                        cursor.close()
+                    logging.info('User#{} uploaded file {} under launchpad bug#{}'.
+                                 format(g.user['id'], final_filename, launchpad_id))
+            else:
+                logging.error('User#{} tried to upload a file with invalid format'.format(g.user['id']))
+                res = make_response(jsonify({"message": "Error: you did not supply a valid file in your request"}), 400)
+                return res
+            # except RequestEntityTooLarge:
+            #     flash(u'Error: File size exceeds the 16GB limit')
+            #     return redirect(oid.get_next_url())
+            res = make_response(jsonify({"message": "file uploaded successfully: {}".format(f.filename)}), 200)
+            return res
+        return render_template('upload.html')
+    except RequestEntityTooLarge:
+        flash(u'Error: File size exceeds the 10GB limit')
+        return redirect(oid.get_next_url())
+
+
+@app.route('/user_files/', methods=['GET', 'POST'])
+def list_user_files():
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    user_files = []
+    if request.method == 'GET':
+        with g.connection.cursor() as cursor:
+            search = request.args.get('search')
+            if search:
+                sql = "SELECT f.*, l.title FROM files f JOIN launchpads l ON f.launchpad_id = l.id " \
+                      "WHERE (launchpad_id = '{}' OR title LIKE '%{}%') AND user_id = {} " \
+                      "ORDER BY launchpad_id DESC, f.name;".format(search, search, g.user['id'])
+            # sql = "SELECT * FROM launchpads WHERE id IN (SELECT DISTINCT launchpad_id FROM files WHERE user_id = %s);"
+                cursor.execute(sql,)
+            else:
+                sql = "SELECT f.*, l.title FROM files f JOIN launchpads l ON f.launchpad_id = l.id " \
+                      "WHERE user_id = %s ORDER BY launchpad_id DESC;"
+                cursor.execute(sql, (g.user['id'],))
+            user_files = cursor.fetchall()
+            cursor.close()
+    return render_template('user_files.html', user_files=user_files)
+
+
+@app.route('/public_files/', methods=['GET', 'POST'])
+def list_public_files():
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    files = []
+    if request.method == 'GET':
+        with g.connection.cursor() as cursor:
+            sql = "SELECT f.*, l.title, f.user_id, u.name AS user_name FROM files f JOIN launchpads l " \
+                  "ON f.launchpad_id = l.id JOIN openid_users u ON f.user_id = u.id;"
+            cursor.execute(sql)
+            files = cursor.fetchall()
+            cursor.close()
+        # if files:
+        #     files = list(map(lambda f: f.update({'editable': (f['user_id'] == g.user)})))
+    return render_template('public_files.html', public_files=files)
+
+
+@app.route('/launchpads/', methods=['GET', 'POST'])
+def list_all_launchpads():
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    if request.method == 'GET':
+        with g.connection.cursor() as cursor:
+            search = request.args.get('search')
+            if search:
+                sql = "SELECT * FROM launchpads WHERE (id = '{}' OR title LIKE '%{}%') " \
+                      "AND id IN (SELECT DISTINCT launchpad_id FROM files) ORDER BY id DESC;".format(search, search)
+            # sql = "SELECT * FROM launchpads WHERE id IN (SELECT DISTINCT launchpad_id FROM files WHERE user_id = %s);"
+                cursor.execute(sql,)
+            else:
+                sql = "SELECT * FROM launchpads WHERE id IN (SELECT DISTINCT launchpad_id FROM files) ORDER BY id DESC;"
+                cursor.execute(sql,)
+            user_launchpads = cursor.fetchall()
+            cursor.close()
+    return render_template('launchpads.html', user_launchpads=user_launchpads)
+
+
+@app.route('/launchpad/<launchpad_id>', methods=['GET', 'POST'])
+def list_files_under_a_launchpad(launchpad_id):
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    if request.method == 'GET':
+        with g.connection.cursor() as cursor:
+            sql = "SELECT f.*, u.name AS user_name FROM files f " \
+                  "JOIN openid_users u ON f.user_id = u.id WHERE launchpad_id = %s;"
+            cursor.execute(sql, (launchpad_id,))
+            launchpad_files = cursor.fetchall()
+            sql = "SELECT * FROM launchpads WHERE id = %s;"
+            cursor.execute(sql, (launchpad_id,))
+            launchpad_info = cursor.fetchone()
+            cursor.close()
+    return render_template('launchpad.html', launchpad_files=launchpad_files, launchpad_info=launchpad_info)
+    # return render_template('user_files.html', user_launchpads=user_launchpads)
+
+
+@app.route('/edit_file/<file_id>', methods=['GET', 'POST'])
+def edit_file(file_id):
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    with g.connection.cursor() as cursor:
+        sql = "SELECT * FROM files WHERE id = %s;"
+        cursor.execute(sql, (file_id,))
+        user_files = cursor.fetchone()
+        form = {'name': user_files['name'], 'launchpad_id': user_files['launchpad_id']}
+        old_form = form.copy()
+        cursor.close()
+    if request.method == 'POST':
+        form['name'] = request.form['name']
+        form['launchpad_id'] = request.form['launchpad_id']
+        if not form['name']:
+            flash(u'Error: you have to provide a name')
+        else:
+            # _dir = os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(g.user['id']))
+            # flash(os.path.join(_dir, old_form['launchpad_id'], old_form['name']))
+            if old_form['name'] != form['name'] or old_form['launchpad_id'] != int(form['launchpad_id']):
+                _dir = os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(g.user['id']))
+                _new_dir = os.path.join(_dir, str(form['launchpad_id']))
+                if old_form['launchpad_id'] != form['launchpad_id']:
+                    with g.connection.cursor() as cursor:
+                        sql = "SELECT * FROM launchpads WHERE id = %s;"
+                        cursor.execute(sql, (form['launchpad_id'],))
+                        launchpad_info = cursor.fetchone()
+                        if not launchpad_info:
+                            try:
+                                launchpad_info = get_launchpad_info(int(form['launchpad_id']))
+                            except ValueError:
+                                res = make_response("Error: Launchpad bug id does not exist", 400)
+                                return res
+                            if launchpad_info == 'Launchpad bug id does not exist':
+                                flash(u'Error: Launchpad bug id does not exist')
+                                return redirect(oid.get_next_url())
+                            elif launchpad_info == 'You need to choose a valid StarlingX Launchpad':
+                                flash(u'Error: You need to choose a valid StarlingX Launchpad')
+                                return redirect(oid.get_next_url())
+                            elif launchpad_info[1] is True:
+                                flash(u'Error: Launchpad bug is closed')
+                                return redirect(oid.get_next_url())
+                            else:
+                                sql = "INSERT INTO launchpads (id, title) VALUES (%s, %s);"
+                                cursor.execute(sql, (form['launchpad_id'], launchpad_info[0],))
+                                cursor.close()
+                            if not os.path.isdir(_new_dir):
+                                os.mkdir(_new_dir)
+                os.rename(os.path.join(_dir, str(old_form['launchpad_id']), old_form['name']),
+                          os.path.join(_dir, str(form['launchpad_id']), form['name']))
+                if old_form['name'] == form['name']:
+                    logging.info('User#{} changed file {}/{} to {}/{}'.
+                                 format(g.user['id'], old_form['launchpad_id'],
+                                        old_form['name'], form['launchpad_id'], form['name']))
+            with g.connection.cursor() as cursor:
+                sql = "UPDATE files SET name = %s, launchpad_id = %s, modified_date = %s WHERE id = %s;"
+                cursor.execute(sql, (form['name'], form['launchpad_id'], datetime.now(), file_id,))
+                cursor.close()
+            flash(u'File information successfully updated')
+            return redirect(url_for('edit_file', file_id=file_id, form=form))
+    return render_template('edit_file.html', file_id=file_id, form=form)
+
+
+@app.route('/delete_file', methods=['GET', 'POST'])
+# @confirmation_required(confirm_delete_file)
+def delete_file():
+    """Updates a profile"""
+    if g.user is None:
+        abort(401)
+    file_id = request.form['id']
+    with g.connection.cursor() as cursor:
+        file_name_sql = "SELECT name, launchpad_id FROM files WHERE id=%s;"
+        cursor.execute(file_name_sql, (file_id,))
+        file_info = cursor.fetchone()
+        os.remove(os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(g.user['id']), str(file_info['launchpad_id']),
+                               file_info['name']))
+
+        sql = "DELETE FROM files WHERE id=%s;"
+        cursor.execute(sql, (file_id,))
+        cursor.close()
+        logging.info('User#{} deleted file {} under launchpad bug#{}'.
+                     format(g.user['id'], secure_filename(file_info['name']), file_info['launchpad_id']))
+        flash(u'File deleted')
+    return redirect(url_for('list_user_files'))
+
+
+@app.route('/download_file/<file_id>', methods=['GET', 'POST'])
+def download_file(file_id):
+    with g.connection.cursor() as cursor:
+        file_name_sql = "SELECT name, launchpad_id, user_id FROM files WHERE id=%s;"
+        cursor.execute(file_name_sql, (file_id,))
+        file_info = cursor.fetchone()
+        download_link = os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(file_info['user_id']),
+                                     str(file_info['launchpad_id']), file_info['name'])
+        cursor.close()
+        return send_file(download_link, attachment_filename=file_info['name'], as_attachment=True, cache_timeout=0)
+
+
+@app.route('/download_launchpad/<launchpad_id>', methods=['GET', 'POST'])
+def download_launchpad(launchpad_id):
+    zipf = zipfile.ZipFile('{}.zip'.format(launchpad_id), 'w', zipfile.ZIP_DEFLATED)
+    with g.connection.cursor() as cursor:
+        launchpad_file_sql = "SELECT f.name, f.user_id, f.launchpad_id, u.name AS uploader FROM files f " \
+                             "JOIN openid_users u ON f.user_id = u.id WHERE launchpad_id=%s;"
+        cursor.execute(launchpad_file_sql, (launchpad_id,))
+        files = cursor.fetchall()
+        for file in files:
+            zipf.write(os.path.join(app.config['BASE_DIR'], 'files/{}/'.format(file['user_id']),
+                                    str(file['launchpad_id']), file['name']),
+                       os.path.join(file['uploader'], file['name']))
+            cursor.close()
+        zipf.close()
+
+        @after_this_request
+        def remove_file(response):
+            try:
+                file_path = '{}.zip'.format(launchpad_id)
+                os.remove(file_path)
+                file_handle = open(file_path, 'r')
+                file_handle.close()
+            except Exception as error:
+                app.logger.error("Error removing or closing downloaded file handle", error)
+            return response
+        return send_file('{}.zip'.format(launchpad_id), mimetype='zip',
+                         attachment_filename='{}.zip'.format(launchpad_id), as_attachment=True, cache_timeout=0)
+
+
+@app.route('/file_exists/', methods=['GET', 'POST'])
+def file_exists():
+    file_name = request.args.get('file_name')
+    launchpad_id = request.args.get('launchpad_id')
+    if not is_allowed(file_name):
+        return '-1'
+    with g.connection.cursor() as cursor:
+        file_name_sql = "SELECT id FROM files WHERE launchpad_id=%s AND name=%s AND user_id=%s;"
+        cursor.execute(file_name_sql, (launchpad_id, secure_filename(file_name), g.user['id']))
+        if cursor.fetchone():
+            return '1'
+        else:
+            return '0'
+
+
+@app.route('/view_log/', methods=['GET', 'POST'])
+def view_log():
+    return send_file('collect.log', mimetype='text/plain')
+
+
+if __name__ == "__main__":
+    app.run(debug=True, host='0.0.0.0')

tools/collect_filesystem/app/requirements.txt

@@ -0,0 +1,2 @@
+Flask==1.1.1
+mysql-connector

tools/collect_filesystem/app/static/main.js

@@ -0,0 +1,415 @@
+function ConfirmDelete(elem) {
+    localStorage.setItem('deleteId', $(elem).attr('data-id'));
+    $('#deleteModal').modal();
+}
+
+function Delete() {
+    $.ajax({
+        url: '/delete_file',
+        data: {
+            id: localStorage.getItem('deleteId')
+        },
+        type: 'POST',
+        success: function(res) {
+            $('#deleteModal').modal('hide');
+            location.reload();
+        },
+        error: function(error) {
+            console.log(error);
+        }
+    });
+}
+
+// Get a reference to the progress bar, wrapper & status label
+var progress_wrapper = document.getElementById("progress_wrapper");
+
+// Get a reference to the 3 buttons
+var upload_btn = document.getElementById("upload_btn");
+var loading_btn = document.getElementById("loading_btn");
+var cancel_btn = document.getElementById("cancel_btn");
+
+// Get a reference to the alert wrapper
+var alert_wrapper = document.getElementById("alert_wrapper");
+
+// Get a reference to the file input element & input label
+var input = document.getElementById("file_input");
+var launchpad_input = document.getElementById("launchpad_input");
+var file_input_label = document.getElementById("file_input_label");
+
+var search_input_label = document.getElementById("search_input_label");
+var launchpad_info = document.getElementById("launchpad_info");
+
+var upload_count = 0;
+
+function search() {
+    search_input = search_input_label.value;
+    location.search = "?search="+search_input;
+}
+
+// Function to show alerts
+function show_alert(message, alert) {
+
+  alert_wrapper.innerHTML = alert_wrapper.innerHTML + `
+    <div id="alert" class="alert alert-${alert} alert-dismissible fade show" role="alert">
+      <span>${message}</span>
+      <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+        <span aria-hidden="true">&times;</span>
+      </button>
+    </div>
+  `
+
+}
+
+function revert() {
+    var tbody = $('table tbody');
+    tbody.html($('tr',tbody).get().reverse());
+    if (document.getElementById("table_order").classList.contains("fa-caret-square-o-down")) {
+        document.getElementById("table_order").classList.remove("fa-caret-square-o-down");
+        document.getElementById("table_order").classList.add("fa-caret-square-o-up");
+    } else {
+        document.getElementById("table_order").classList.remove("fa-caret-square-o-up");
+        document.getElementById("table_order").classList.add("fa-caret-square-o-down");
+    }
+}
+
+function upload() {
+
+  upload_count = 0
+
+  // Reject if the file input is empty & throw alert
+  if (!input.value) {
+
+    show_alert("No file selected", "warning")
+
+    return;
+
+  }
+
+  var request = new XMLHttpRequest();
+  var launchpad_id = launchpad_input.value;
+
+  request.open("get", "http://128.224.141.2:5000/check_launchpad/"+launchpad_id);
+  request.send();
+
+  // Clear any existing alerts
+  alert_wrapper.innerHTML = "";
+
+  request.addEventListener("load", function (e) {
+
+    if (request.status == 200) {
+
+      // Hide the upload button
+      upload_btn.classList.add("d-none");
+
+      // Show the loading button
+      loading_btn.classList.remove("d-none");
+
+      // Show the cancel button
+      cancel_btn.classList.remove("d-none");
+
+      // Show the progress bar
+      progress_wrapper.classList.remove("d-none");
+
+      // Show the progress bar
+      launchpad_info.classList.remove("d-none");
+
+      launchpad_info.innerHTML = 'Launchpad title: '+request.response;
+
+      // Disable the input during upload
+      input.disabled = true;
+      launchpad_input.disabled = true;
+
+      progress_wrapper.innerHTML = ""
+
+      for (var i = 0; i < input.files.length; i++) {
+        progress_wrapper.innerHTML = progress_wrapper.innerHTML + `
+          <div id="progress_wrapper_${i}">
+            <label id="progress_status_${i}">Initializing upload...</label>
+            <button type="button" id="cancel_btn_${i}" class="btn btn-secondary btn-sm">Cancel</button>
+            <button type="button" id="ignore_btn_${i}" class="btn btn-secondary btn-sm d-none">Cancel</button>
+            <button type="button" id="overwrite_btn_${i}" class="btn btn-danger btn-sm d-none">Overwrite</button>
+            <button type="button" id="rename_btn_${i}" class="btn btn-primary btn-sm d-none">Rename</button>
+            <div class="progress mb-3">
+              <div id="progress_${i}" class="progress-bar" role="progressbar" aria-valuenow="25" aria-valuemin="0" aria-valuemax="100"></div>
+            </div>
+          </div>`
+      }
+
+      for (var i = 0; i < input.files.length; i++) {
+        upload_single_file(input.files[i], i);
+      }
+
+    }
+    else {
+
+      // Reset the input placeholder
+      file_input_label.innerText = "Select file or drop it here to upload";
+      launchpad_input.innerText = "";
+
+      show_alert(`${request.response}`, "danger");
+
+    }
+
+  });
+
+//  reset();
+}
+
+// Function to upload single file
+function upload_single_file(file, i) {
+
+  var progress_wrapper_single = document.getElementById(`progress_wrapper_${i}`);
+  var progress = document.getElementById(`progress_${i}`);
+  var progress_status = document.getElementById(`progress_status_${i}`);
+
+  var cancel_btn_single = document.getElementById(`cancel_btn_${i}`);
+  var ignore_btn_single = document.getElementById(`ignore_btn_${i}`);
+  var overwrite_btn_single = document.getElementById(`overwrite_btn_${i}`);
+  var rename_btn_single = document.getElementById(`rename_btn_${i}`);
+
+  var url = "http://128.224.141.2:5000/upload/"
+
+  // Create a new FormData instance
+  var data = new FormData();
+
+  // Create a XMLHTTPRequest instance
+  var request = new XMLHttpRequest();
+  var request_file_check = new XMLHttpRequest();
+
+  // Set the response type
+  request.responseType = "json";
+
+  // Get a reference to the files
+//  var file = input.files[0];
+  // Get a reference to the launchpad id
+  var launchpad_id = launchpad_input.value;
+
+//  // Get a reference to the filename
+//  var filename = file.name;
+
+  // Get a reference to the filesize & set a cookie
+//  var filesize = file.size;
+//  document.cookie = `filesize=${filesize}`;
+
+  // Append the file to the FormData instance
+//  data.append("file", file);
+
+  request_file_check.open("get", '/file_exists/?launchpad_id='+launchpad_id+'&file_name='+file.name);
+  request_file_check.send();
+
+  request_file_check.addEventListener("load", function (e) {
+    if (request_file_check.responseText == '0'){
+      // Open and send the request
+      request.open("post", url+"?launchpad_id="+launchpad_id);
+      data = new FormData();
+      data.append("file", file);
+      request.send(data);
+    } else if (request_file_check.responseText == '1'){
+      progress_status.innerText = `File already exists: ${file.name}`;
+      progress_status.style.color = 'red';
+      cancel_btn_single.classList.add("d-none");
+      ignore_btn_single.classList.remove("d-none");
+      overwrite_btn_single.classList.remove("d-none");
+      rename_btn_single.classList.remove("d-none");
+    } else {
+      show_alert('Error: you did not supply a valid file in your request', "warning");
+      upload_count++;
+
+      if (upload_count == input.files.length){
+        reset();
+      }
+    }
+  });
+
+  ignore_btn_single.addEventListener("click", function () {
+
+    progress_status.style.color = 'black';
+
+    cancel_btn_single.classList.remove("d-none");
+    ignore_btn_single.classList.add("d-none");
+    overwrite_btn_single.classList.add("d-none");
+    rename_btn_single.classList.add("d-none");
+
+    show_alert(`Upload cancelled: ${file.name}`, "primary");
+
+    progress_wrapper_single.classList.add("d-none");
+
+    upload_count++;
+
+    if (upload_count == input.files.length){
+      reset();
+    }
+
+  })
+
+  overwrite_btn_single.addEventListener("click", function () {
+
+    progress_status.style.color = 'black';
+
+    cancel_btn_single.classList.remove("d-none");
+    ignore_btn_single.classList.add("d-none");
+    overwrite_btn_single.classList.add("d-none");
+    rename_btn_single.classList.add("d-none");
+
+    request.open("post", url+"?launchpad_id="+launchpad_id+'&conflict=0');
+    data = new FormData();
+    data.append("file", file);
+    request.send(data);
+
+  })
+
+  rename_btn_single.addEventListener("click", function () {
+
+    progress_status.style.color = 'black';
+
+    cancel_btn_single.classList.remove("d-none");
+    ignore_btn_single.classList.add("d-none");
+    overwrite_btn_single.classList.add("d-none");
+    rename_btn_single.classList.add("d-none");
+
+    request.open("post", url+"?launchpad_id="+launchpad_id+'&conflict=1');
+    data = new FormData();
+    data.append("file", file);
+    request.send(data);
+
+  })
+
+  // request progress handler
+  request.upload.addEventListener("progress", function (e) {
+
+    // Get the loaded amount and total filesize (bytes)
+    var loaded = e.loaded;
+    var total = e.total;
+
+    // Calculate percent uploaded
+    var percent_complete = (loaded / total) * 100;
+
+    // Update the progress text and progress bar
+    progress.setAttribute("style", `width: ${Math.floor(percent_complete)}%`);
+    progress_status.innerText = `${Math.floor(percent_complete)}% uploaded: ${file.name}`;
+
+    if (loaded == total) {
+      progress_status.innerText = `Saving file: ${file.name}`;
+    }
+
+  })
+
+  // request load handler (transfer complete)
+  request.addEventListener("load", function (e) {
+
+    if (request.status == 200) {
+
+      show_alert(`${request.response.message}`, "success");
+
+    }
+    else {
+
+      show_alert(`${request.response.message}`, "danger");
+
+    }
+
+    progress_wrapper_single.classList.add("d-none");
+
+    upload_count++;
+
+    if (upload_count == input.files.length){
+      reset();
+    }
+
+  });
+
+  // request error handler
+  request.addEventListener("error", function (e) {
+
+    show_alert(`Error uploading file: ${file.name}`, "warning");
+
+    progress_wrapper_single.classList.add("d-none");
+
+    upload_count++;
+
+    if (upload_count == input.files.length){
+      reset();
+    }
+
+  });
+
+  // request abort handler
+  request.addEventListener("abort", function (e) {
+
+    show_alert(`Upload cancelled: ${file.name}`, "primary");
+
+    progress_wrapper_single.classList.add("d-none");
+
+    upload_count++;
+
+    if (upload_count == input.files.length){
+      reset();
+    }
+
+  });
+
+  cancel_btn.addEventListener("click", function () {
+
+    request.abort();
+
+  })
+
+  cancel_btn_single.addEventListener("click", function () {
+
+    request.abort();
+
+  })
+
+}
+
+// Function to update the input placeholder
+function input_filename() {
+//    file_input_label.innerText = typeof input.files;
+//    var all_files = input.files.values().reduce(function (accumulator, file) {
+//      return accumulator + file.name;
+//    }, 0);
+
+    var all_files = input.files[0].name;
+
+    for (var i = 1; i < input.files.length; i++){
+        all_files = all_files + ', ' + input.files[i].name
+    }
+    file_input_label.innerText = all_files;
+
+//    file_input_label.innerText = input.files[0].name;
+//    file_input_label.innerText = input.files.toString();
+
+}
+
+// Function to reset the page
+function reset() {
+
+  // Clear the input
+  input.value = null;
+
+  // Hide the cancel button
+  cancel_btn.classList.add("d-none");
+
+  // Reset the input element
+  input.disabled = false;
+  launchpad_input.disabled = false;
+
+  // Show the upload button
+  upload_btn.classList.remove("d-none");
+
+  // Hide the loading button
+  loading_btn.classList.add("d-none");
+
+  // Hide the progress bar
+  progress_wrapper.classList.add("d-none");
+
+  // Reset the input placeholder
+  file_input_label.innerText = "Select file or drop it here to upload";
+  launchpad_input.innerText = "";
+
+  // Show the progress bar
+  launchpad_info.classList.add("d-none");
+
+  launchpad_info.innerHTML = "";
+
+}

tools/collect_filesystem/app/static/style.css

@@ -0,0 +1,46 @@
+body {
+    font-family: 'Georgia', serif;
+    font-size: 16px;
+    margin: 30px;
+    padding: 0;
+}
+
+a {
+    color: #335E79;
+}
+
+p.message {
+    color: #335E79;
+    padding: 10px;
+    background: #CADEEB;
+}
+
+p.error {
+    color: #783232;
+    padding: 10px;
+    background: #EBCACA;
+}
+
+input {
+    font-family: 'Georgia', serif;
+    font-size: 16px;
+    border: 1px solid black;
+    color: #335E79;
+    padding: 2px;
+}
+
+input[type="submit"] {
+    background: #CADEEB;
+    color: #335E79;
+    border-color: #335E79;
+}
+
+#basic-addon3 {
+    background: url(openid.png) 4px no-repeat;
+    background-color: #e9ecef;
+    padding-left: 24px;
+}
+
+h1, h2 {
+    font-weight: normal;
+}

tools/collect_filesystem/app/templates/413.html

@@ -0,0 +1,7 @@
+{% extends "layout.html" %}
+{% block title %}Page Not Found{% endblock %}
+{% block body %}
+  <h1>Page Not Found</h1>
+  <p>What you were looking for is just not there.
+  <p><a href="{{ url_for('index') }}">go somewhere nice</a>
+{% endblock %}

tools/collect_filesystem/app/templates/_confirm.html

@@ -0,0 +1,7 @@
+<body>
+<h1>{{ desc }}</h1>
+<form action="{{ action_url }}" method="GET">
+    <input type="hidden" name="confirm" value="1">
+    <input type="submit" value="Yes">
+</form>
+</body>

tools/collect_filesystem/app/templates/bak/upload.html

@@ -0,0 +1,51 @@
+{% extends "layout.html" %}
+{% block title %}Sign in{% endblock %}
+{% block body %}
+<h2>Upload</h2>
+
+<form action = "" method = "POST" enctype = "multipart/form-data">
+    <div class="form-group mb-3">
+        <div class="custom-file">
+            <input type="file" class="custom-file-input" name="file_input" id="file_input" oninput="input_filename();">
+            <label id="file_input_label" class="custom-file-label">Select file</label>
+            <div>Launchpad ID:</div>
+            <div><input type=text name=launchpad_id size=30 value=""></div>
+        </div>
+    </div>
+
+<!--    <div class="form-group mb-3">-->
+<!--        <div class="custom-file">-->
+<!--&lt;!&ndash;            <input type = "file" name = "file" />&ndash;&gt;-->
+<!--            <input type="file" class="custom-file-input" name="file_input" id="file_input" oninput="input_filename();">-->
+<!--            <label id="file_input_label" class="custom-file-label">Select file</label>-->
+<!--            <div>Launchpad ID:</div>-->
+<!--            <div><input type=text name=launchpad_id size=30 value=""></div>-->
+<!--        </div>-->
+<!--    </div>-->
+
+    <button onclick="upload('{{ request.url }}');" id="upload_btn" class="btn btn-primary">Upload</button>
+    <!--        <input type = "submit"/>-->
+
+    <button class="btn btn-primary d-none" id="loading_btn" type="button" disabled>
+        <span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span>
+        Uploading...
+    </button>
+
+    <button type="button" id="cancel_btn" class="btn btn-secondary d-none">Cancel upload</button>
+</form>
+
+<div class="modal fade" id="overwriteModal" tabindex="-1" role="dialog" aria-labelledby="overwriteModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header" style="text-align:center;">
+                <h4 class="modal-title" style="color:red;" id="overwriteModalLabel">Are you sure you want to overwrite this file?</h4>
+            </div>
+
+            <div class="modal-footer">
+                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
+                <button type="button" class="btn btn-primary" onclick="Overwrite()">Overwrite</button>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}

tools/collect_filesystem/app/templates/create_profile.html

@@ -0,0 +1,28 @@
+{% extends "layout.html" %}
+{% block title %}Create Profile{% endblock %}
+{% block body %}
+  <h2>Create Profile</h2>
+  <p>
+    Hey!  This is the first time you signed in on this website.  In
+    order to proceed we need some extra information from you:
+  <form action="" method=post>
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text">Name:</span>
+      </div>
+      <input type="text" name=name class="form-control" placeholder="name" aria-label="name" aria-describedby="basic-addon1" value="{{ request.values.name }}">
+    </div>
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text" id="basic-addon1">E-Mail:</span>
+      </div>
+      <input type="text" name=email class="form-control" placeholder="email" aria-label="email" aria-describedby="basic-addon1" value="{{ request.values.email }}">
+    </div>
+    <p>
+      <input type=submit value="Create profile" class="btn btn-primary">
+      <input type=hidden name=next value="{{ next }}">
+  </form>
+  <p>
+    If you don't want to proceed, you can <a href="{{ url_for('logout')
+    }}">sign out</a> again.
+{% endblock %}

tools/collect_filesystem/app/templates/edit_file.html

@@ -0,0 +1,27 @@
+{% extends "layout.html" %}
+{% block title %}Edit Profile{% endblock %}
+{% block body %}
+  <h2>Edit File</h2>
+  <form action="" method=post>
+<!--    <dl>-->
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text">Name:</span>
+      </div>
+      <input type="text" name=name class="form-control" placeholder="name" aria-label="name" aria-describedby="basic-addon1" value="{{ form.name }}">
+    </div>
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text" id="basic-addon1">Launchpad ID:</span>
+      </div>
+      <input type="text" name=launchpad_id class="form-control" placeholder="launchpad_id" aria-label="launchpad_id" aria-describedby="basic-addon1" value="{{ form.launchpad_id }}">
+    </div>
+<!--      <dt>Name:-->
+<!--      <dd><input type=text name=name size=30 value="{{ form.name }}">-->
+<!--      <dt>Launchpad ID:-->
+<!--      <dd><input type=text name=launchpad_id size=30 value="{{ form.launchpad_id }}">-->
+<!--    </dl>-->
+    <p>
+      <input type=submit value="Update file" class="btn btn-primary">
+  </form>
+{% endblock %}

tools/collect_filesystem/app/templates/edit_profile.html

@@ -0,0 +1,22 @@
+{% extends "layout.html" %}
+{% block title %}Edit Profile{% endblock %}
+{% block body %}
+  <h2>Edit Profile</h2>
+  <form action="" method=post>
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text">Name:</span>
+      </div>
+      <input type="text" name=name class="form-control" placeholder="name" aria-label="name" aria-describedby="basic-addon1" value="{{ form.name }}">
+    </div>
+    <div class="input-group mb-3">
+      <div class="input-group-prepend">
+        <span class="input-group-text" id="basic-addon1">E-Mail:</span>
+      </div>
+      <input type="text" name=email class="form-control" placeholder="email" aria-label="email" aria-describedby="basic-addon1" value="{{ form.email }}">
+    </div>
+    <p>
+      <input type=submit value="Update profile" class="btn btn-primary">
+      <input type=submit name=delete value="Delete" class="btn btn-primary">
+  </form>
+{% endblock %}

tools/collect_filesystem/app/templates/index.html

@@ -0,0 +1,10 @@
+{% extends "layout.html" %}
+{% block body %}
+  <h2>Overview</h2>
+  {% if g.user %}
+  <p>
+    Hello {{ g.user.name }}!
+  {% endif %}
+  <p>
+    This is just an example page so that something is here.
+{% endblock %}

tools/collect_filesystem/app/templates/launchpad.html

@@ -0,0 +1,40 @@
+{% extends "layout.html" %}
+{% block body %}
+  <h2>
+    {{ launchpad_info['title'] }}
+    <button class="btn btn-secondary" onclick="window.open(`https://bugs.launchpad.net/bugs/{{ launchpad_info['id'] }}`, '_blank')">
+        Go to the launchpad page
+    </button>
+    <button class="btn btn-secondary" onclick="window.location.href=`{{ url_for('download_launchpad', launchpad_id=launchpad_info['id']) }}`">
+        Download all files
+    </button>
+  </h2>
+  <table class="table table-striped" style="width:100%">
+    <thead>
+      <th>Name</th>
+      <th>Uploaded by</th>
+      <th>Last Modified</th>
+      <th></th>
+    </thead>
+    {% for launchpad_file in launchpad_files %}
+    <tr>
+      <td><a href="{{ url_for('download_file', file_id=launchpad_file['id']) }}">{{ launchpad_file['name'] }}</a></td>
+      <td>{{ launchpad_file['user_name'] }} </td>
+      <td>{{ launchpad_file['modified_date'] }} </td>
+      {% if g.user['id'] == launchpad_file['user_id'] %}
+      <td>
+    <!--        <span><a data-id=${launchpad_file['id']} onclick="file_id(this)" >Edit</a></span>-->
+        <button data-id="{{ launchpad_file['id'] }}" class="btn btn-secondary btn-sm" onclick="window.location.href=`{{ url_for('edit_file', file_id=launchpad_file['id']) }}`">
+            Edit
+        </button>
+    <!--        <span><a data-id=${Id} onclick="ConfirmDelete(this)" ><span class="glyphicon glyphicon-trash"></span></a></span>-->
+    <!--        <span><a href="{{ url_for('delete_file', file_id=launchpad_file['id']) }}">Delete</a></span>-->
+        <button data-id="{{ launchpad_file['id'] }}" onclick="ConfirmDelete(this)" class="btn btn-danger btn-sm" >Delete</button>
+      </td>
+      {% else %}
+      <td></td>
+      {% endif %}
+    </tr>
+    {% endfor %}
+  </table>
+{% endblock %}

tools/collect_filesystem/app/templates/launchpads.html

@@ -0,0 +1,44 @@
+{% extends "layout.html" %}
+{% block body %}
+  <h2>Launchpads</h2>
+  <div>
+      <div class="input-group">
+        <input type="text" class="form-control" placeholder="Search this blog" id="search_input_label">
+        <div class="input-group-append">
+          <button class="btn btn-secondary" type="button" onclick="search();">
+            <i class="fa fa-search"></i>
+          </button>
+        </div>
+      </div>
+  </div>
+  {% if user_launchpads|length %}
+  <table style="width:100%" class="table table-striped">
+    <thead>
+    <tr>
+      <th onclick="revert();" style="cursor: pointer;" title="Click to sort the launchpads">
+          LP Files <i id="table_order" class="fa fa-caret-square-o-down"></i>
+      </th>
+      <th>Launchpad Link</th>
+      <th>Launchpad Title</th>
+    </tr>
+    </thead>
+      {% for user_launchpad in user_launchpads %}
+        <tr>
+        <!--      <td><a href="{{ url_for('download_file', file_id=user_launchpad['id']) }}">{{ user_launchpad['title'] }}</a></td>-->
+          <td>
+            <a href="{{ url_for('list_files_under_a_launchpad', launchpad_id=user_launchpad['id']) }}">{{ user_launchpad['id'] }}</a>
+          </td>
+          <td>
+            <a href="https://bugs.launchpad.net/bugs/{{ user_launchpad['id'] }}">https://bugs.launchpad.net/bugs/{{ user_launchpad['id'] }}</a>
+          </td>
+          <td>{{ user_launchpad['title'] }}</td>
+        </tr>
+      {% endfor %}
+  </table>
+  {% else %}
+  <div>Your search returns no result</div>
+  {% endif %}
+
+  <!-- Import Bootstrap JavaScript here -->
+  <script type=text/javascript src="{{url_for('static', filename='main.js') }}"></script>
+{% endblock %}

tools/collect_filesystem/app/templates/layout.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<title>{% block title %}Welcome{% endblock %} | Flask OpenID Example</title>
+<link rel=stylesheet type=text/css href="{{ url_for('static',
+  filename='style.css') }}">
+<script src="https://code.jquery.com/jquery-3.4.1.js" integrity="sha384-mlceH9HlqLp7GMKHrj5Ara1+LvdTZVMx4S1U43/NxCvAkzIo8WJ0FE7duLel3wVo" crossorigin="anonymous"></script>
+<!--<script type="text/javascript" src="http://ajax.aspnetcdn.com/ajax/jquery.templates/beta1/jquery.tmpl.js"></script>-->
+<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
+<h1>Collect</h1>
+<ul class=navigation>
+  <li><a href="{{ url_for('index') }}">overview</a>
+  {% if g.user %}
+  <li><a href="{{ url_for('edit_profile') }}">profile</a>
+  <li><a href="{{ url_for('upload') }}">upload</a>
+  <li><a href="{{ url_for('list_user_files') }}">personal files</a>
+  <li><a href="{{ url_for('list_public_files') }}">public files</a>
+  <li><a href="{{ url_for('list_all_launchpads') }}">launchpads</a>
+  <li><a href="{{ url_for('logout') }}">sign out [{{ g.user.name }}]</a>
+  {% else %}
+  <li><a href="{{ url_for('login') }}">sign in</a>
+  {% endif %}
+</ul>
+{% for message in get_flashed_messages() %}
+  <p class=message>{{ message }}
+{% endfor %}
+{% block body %}{% endblock %}

tools/collect_filesystem/app/templates/login.html

@@ -0,0 +1,18 @@
+{% extends "layout.html" %}
+{% block title %}Sign in{% endblock %}
+{% block body %}
+  <h2>Sign in</h2>
+  <form action="" method=post>
+    {% if error %}<p class=error><strong>Error:</strong> {{ error }}</p>{% endif %}
+    <p>
+      Enter your Ubuntu One OpenID:
+      <div class="input-group mb-3">
+        <div class="input-group-prepend">
+          <span class="input-group-text" id="basic-addon3">https://launchpad.net/~</span>
+        </div>
+        <input type=text class="form-control" name=openid size=30 aria-describedby="basic-addon3">
+      </div>
+      <input type=submit value="Sign in">
+      <input type=hidden name=next value="{{ next }}">
+  </form>
+{% endblock %}

tools/collect_filesystem/app/templates/public_files.html

@@ -0,0 +1,64 @@
+{% extends "layout.html" %}
+{% block body %}
+<!--<script type=text/javascript">-->
+<!--  function ConfirmDelete(elem) {-->
+<!--    document.getElementById("demo").innerHTML = "Hello World";-->
+<!--    localStorage.setItem('deleteId', $(elem).attr('data-id'));-->
+<!--    $('#deleteModal').modal();-->
+<!--}-->
+<!--</script>-->
+    <h2>Files</h2>
+    {% if public_files|length %}
+    <table class="table table-striped" style="width:100%">
+    <thead>
+        <tr>
+          <th>Name</th>
+          <th>Launchpad</th>
+          <th>Uploaded by</th>
+          <th>Last Modified</th>
+          <th></th>
+        </tr>
+    </thead>
+    {% for public_file in public_files %}
+    <tr>
+      <td><a href="{{ url_for('download_file', file_id=public_file['id']) }}">{{ public_file['name'] }}</a></td>
+      <td>
+        <a href="{{ url_for('list_files_under_a_launchpad', launchpad_id=public_file['launchpad_id']) }}">{{ public_file['title'] }} #{{ public_file['launchpad_id'] }}</a>
+      </td>
+      <td>{{ public_file['user_name'] }} </td>
+      <td>{{ public_file['modified_date'] }} </td>
+      {% if g.user['id'] == public_file['user_id'] %}
+      <td>
+    <!--        <span><a data-id=${public_file['id']} onclick="file_id(this)" >Edit</a></span>-->
+        <button data-id="{{ public_file['id'] }}" class="btn btn-secondary btn-sm" onclick="window.location.href=`{{ url_for('edit_file', file_id=public_file['id']) }}`">
+            Edit
+        </button>
+    <!--        <span><a data-id=${Id} onclick="ConfirmDelete(this)" ><span class="glyphicon glyphicon-trash"></span></a></span>-->
+    <!--        <span><a href="{{ url_for('delete_file', file_id=public_file['id']) }}">Delete</a></span>-->
+        <button data-id="{{ public_file['id'] }}" onclick="ConfirmDelete(this)" class="btn btn-danger btn-sm" >Delete</button>
+      </td>
+      {% else %}
+      <td></td>
+      {% endif %}
+    </tr>
+    {% endfor %}
+    </table>
+    {% else %}
+    <div>Opps! There is no file here</div>
+    {% endif %}
+    <div class="modal fade" id="deleteModal" tabindex="-1" role="dialog" aria-labelledby="deleteModalLabel" aria-hidden="true">
+        <div class="modal-dialog">
+            <div class="modal-content">
+                <div class="modal-header" style="text-align:center;">
+                    <h4 class="modal-title" style="color:red;" id="deleteModalLabel">Are you sure you want to delete this file?</h4>
+                </div>
+
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
+                    <button type="button" class="btn btn-primary" onclick="Delete()">Delete</button>
+                </div>
+            </div>
+        </div>
+    </div>
+  <script type=text/javascript src="{{url_for('static', filename='main.js') }}"></script>
+{% endblock %}

tools/collect_filesystem/app/templates/upload.html

@@ -0,0 +1,58 @@
+{% extends "layout.html" %}
+{% block title %}Sign in{% endblock %}
+{% block body %}
+
+<body>
+
+  <div class="container">
+    <div class="row">
+      <div class="col">
+
+        <div class="mb-3 mt-3">
+
+          <h2 class="mb-3" style="font-weight: 300">Upload Log Files</h2>
+
+          <div class="form-group mb-3">
+            <div class="custom-file">
+                <input type="file" class="custom-file-input" name="file_input" id="file_input" oninput="input_filename();" multiple>
+                <label id="file_input_label" class="custom-file-label" style="text-overflow: ellipsis; overflow: hidden; white-space: nowrap;">Select file or drop it here to upload</label>
+<!--                <div class="input-group-prepend">-->
+<!--                  <span class="input-group-text" id="basic-addon1">Launchpad ID:</span>-->
+<!--                </div>-->
+<!--                <input type="text" name="launchpad_id" class="form-control" id="launchpad_input" size=30 value="">-->
+            </div>
+          </div>
+
+          <div class="input-group mb-3">
+            <div class="input-group-prepend">
+              <span class="input-group-text" id="basic-addon1">Launchpad ID:</span>
+            </div>
+            <input type="text" name="launchpad_id" id="launchpad_input" class="form-control" placeholder="launchpad_input" aria-label="launchpad_input" aria-describedby="basic-addon1">
+          </div>
+
+          <p id="launchpad_info" style="color:blue" class="d-none"></p>
+
+          <button onclick="upload();" id="upload_btn" class="btn btn-primary">Upload</button>
+
+          <button class="btn btn-primary d-none" id="loading_btn" type="button" disabled>
+            <span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span>
+            Uploading...
+          </button>
+
+          <button type="button" id="cancel_btn" class="btn btn-secondary d-none">Cancel upload</button>
+
+        </div>
+
+        <div id="progress_wrapper" class="d-none"></div>
+
+        <div id="alert_wrapper"></div>
+
+      </div>
+    </div>
+  </div>
+
+  <!-- Import Bootstrap JavaScript here -->
+  <script type=text/javascript src="{{url_for('static', filename='main.js') }}"></script>
+
+</body>
+{% endblock %}

tools/collect_filesystem/app/templates/user_files.html

@@ -0,0 +1,72 @@
+{% extends "layout.html" %}
+{% block body %}
+<!--<script type=text/javascript">-->
+<!--  function ConfirmDelete(elem) {-->
+<!--    document.getElementById("demo").innerHTML = "Hello World";-->
+<!--    localStorage.setItem('deleteId', $(elem).attr('data-id'));-->
+<!--    $('#deleteModal').modal();-->
+<!--}-->
+<!--</script>-->
+    <h2>Files</h2>
+  <div>
+      <div class="input-group">
+        <input type="text" class="form-control" placeholder="Search this blog" id="search_input_label">
+        <div class="input-group-append">
+          <button class="btn btn-secondary" type="button" onclick="search();">
+            <i class="fa fa-search"></i>
+          </button>
+        </div>
+      </div>
+  </div>
+    {% if user_files|length %}
+    <table class="table table-striped" style="width:100%">
+    <thead>
+        <tr>
+          <th>Name</th>
+          <th>Launchpad</th>
+          <th onclick="revert();" style="cursor: pointer;" title="Click to sort the files by launchpad id">
+              LP Files <i id="table_order" class="fa fa-caret-square-o-down"></i>
+          </th>
+          <th>Last Modified</th>
+          <th></th>
+        </tr>
+    </thead>
+    {% for user_file in user_files %}
+    <tr>
+      <td><a href="{{ url_for('download_file', file_id=user_file['id']) }}">{{ user_file['name'] }}</a></td>
+      <td>{{ user_file['title'] }}</td>
+      <td>
+        <a href="{{ url_for('list_files_under_a_launchpad', launchpad_id=user_file['launchpad_id']) }}">#{{ user_file['launchpad_id'] }}</a>
+      </td>
+      <td>{{ user_file['modified_date'] }} </td>
+      <td>
+    <!--        <span><a data-id=${user_file['id']} onclick="file_id(this)" >Edit</a></span>-->
+        <button data-id="{{ user_file['id'] }}" class="btn btn-secondary btn-sm" onclick="window.location.href=`{{ url_for('edit_file', file_id=user_file['id']) }}`">
+            Edit
+        </button>
+    <!--        <span><a data-id=${Id} onclick="ConfirmDelete(this)" ><span class="glyphicon glyphicon-trash"></span></a></span>-->
+    <!--        <span><a href="{{ url_for('delete_file', file_id=user_file['id']) }}">Delete</a></span>-->
+        <button data-id="{{ user_file['id'] }}" onclick="ConfirmDelete(this)" class="btn btn-danger btn-sm" >Delete</button>
+      </td>
+    </tr>
+    {% endfor %}
+    </table>
+    {% else %}
+    <div>You have not upload anything yet</div>
+    {% endif %}
+    <div class="modal fade" id="deleteModal" tabindex="-1" role="dialog" aria-labelledby="deleteModalLabel" aria-hidden="true">
+        <div class="modal-dialog">
+            <div class="modal-content">
+                <div class="modal-header" style="text-align:center;">
+                    <h4 class="modal-title" style="color:red;" id="deleteModalLabel">Are you sure you want to delete this file?</h4>
+                </div>
+
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
+                    <button type="button" class="btn btn-primary" onclick="Delete()">Delete</button>
+                </div>
+            </div>
+        </div>
+    </div>
+  <script type=text/javascript src="{{url_for('static', filename='main.js') }}"></script>
+{% endblock %}

tools/collect_filesystem/db/init.sql

@@ -0,0 +1,7 @@
+CREATE DATABASE collect;
+USE collect;
+CREATE TABLE openid_users(id INT not null AUTO_INCREMENT, name VARCHAR(60), email VARCHAR(200), openid VARCHAR(200), PRIMARY KEY (id));
+CREATE TABLE launchpads(id INT not null, title VARCHAR(200), PRIMARY KEY (id));
+CREATE TABLE files(id INT not null AUTO_INCREMENT, name VARCHAR(60), user_id INT not null, launchpad_id INT not null, modified_date TIMESTAMP, PRIMARY KEY (id), file_size FLOAT NOT NULL,
+FOREIGN KEY (user_id) REFERENCES openid_users(id) ON DELETE CASCADE,
+FOREIGN KEY (launchpad_id) REFERENCES launchpads(id) ON DELETE CASCADE);

tools/collect_filesystem/docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+services:
+  app:
+    container_name: collect
+    build: .
+    links:
+      - db
+    ports:
+      - "5000:5000"
+
+  db:
+    image: mysql:latest
+    container_name: collect_db
+    ports:
+      - "32000:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: Wind2019
+    volumes:
+      - ./db:/docker-entrypoint-initdb.d/:ro

tools/collect_filesystem/start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker system prune
+docker-compose up -d

tools/collect_filesystem/update.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker tag docker_app:latest nchen1windriver/collect:trail
+docker push nchen1windriver/collect:trail