Add sqlite for statistics / auditing for each transaction

This allows for a built in audit database for user actions
We can also see how often commands are run, how long they take
as well as who and when they're run

TODO: use sqlcipher to encrypt at rest & password protect the db

Change-Id: Ic7c8927bcfdd46ede3fe6a5aca4f57892ca3f3d4

.gitignore

@@ -15,6 +15,9 @@ dist
 etc/*.pem
 etc/*.json
 
+# sqlite database files
+sqlite/*.db
+
 # Only exists if Bazel was run
 /bazel-out
 

docs/source/developers.md

@@ -12,7 +12,7 @@ Clone the Airship UI repository and build.
     make # Note running behind a proxy can cause issues, notes on solving is in the Appendix
 
 **NOTE:** Make will install node.js-v12.16.3 into your tools directory and will use that as the node binary for the UI
-building, testing and linting.  For windows this can be done using [cygwin](https://www.cygwin.com/) make.
+building, testing and linting.  For windows this can be done using [cygwin](https://www.cygwin.com/) make.  Windows may also require [tdm-gcc](https://jmeubank.github.io/tdm-gcc/) for the sqlite dependency.
 
 Run the airshipui binary
 
@@ -135,6 +135,28 @@ it:
 
     export NODE_EXTRA_CA_CERTS=/<path>/<truststore>.pem
 
+## Issues with SQLITE on Windows
+You may experience issues when attempting to install SQLITE:
+```
+C:\<path>\sqlite> go get github.com/mattn/go-sqlite3
+# github.com/mattn/go-sqlite3
+/usr/lib/gcc/x86_64-pc-cygwin/10/../../../../x86_64-pc-cygwin/bin/ld: cannot find -lmingwex
+/usr/lib/gcc/x86_64-pc-cygwin/10/../../../../x86_64-pc-cygwin/bin/ld: cannot find -lmingw32
+collect2: error: ld returned 1 exit status
+go: failed to remove work dir: GetFileInformationByHandle C:\Users\someUser\AppData\Local\Temp\go-build323470906\NUL: Incorrect function.
+```
+
+To fix this you will need to install [tdm-gcc](https://jmeubank.github.io/tdm-gcc/) and set your path to reference the tdm-gcc first on the path:
+```
+C:\<path>\sqlite> set PATH=c:\TDM-GCC-64\bin;%PATH%
+```
+Test that the tdm-gcc is first on the path
+```
+C:\<path>\sqlite> which gcc
+/cygdrive/c/TDM-GCC-64/bin/gcc
+```
+You should be able to sucessfully run a 'go get github.com/mattn/go-sqlite3' without error
+
 ### Optional proxy settings
 
 #### Environment settings for wget or curl

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.2
+	github.com/mattn/go-sqlite3 v1.14.3
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -706,6 +706,8 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
+github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

pkg/commands/root.go

@@ -24,6 +24,7 @@ import (
 	"opendev.org/airship/airshipui/pkg/configs"
 	"opendev.org/airship/airshipui/pkg/ctl"
 	"opendev.org/airship/airshipui/pkg/log"
+	"opendev.org/airship/airshipui/pkg/statistics"
 	"opendev.org/airship/airshipui/pkg/webservice"
 )
 
@@ -69,6 +70,9 @@ func launch(cmd *cobra.Command, args []string) {
 		log.Fatalf("config %s", err)
 	}
 
+	// Start the statistics database
+	statistics.Init()
+
 	// allows for the circular reference to the webservice package to be broken and allow for the sending
 	// of arbitrary messages from any package to the websocket
 	ctl.Init()

pkg/statistics/recorder.go

@@ -0,0 +1,152 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package statistics
+
+import (
+	"database/sql"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3" // this is required for the sqlite driver
+	"opendev.org/airship/airshipui/pkg/configs"
+	"opendev.org/airship/airshipui/pkg/log"
+)
+
+// Transaction will record the details of the CTL transaction and record them to the DB
+type Transaction struct {
+	Table        configs.WsComponentType
+	SubComponent configs.WsSubComponentType
+	User         *string
+	Started      int64
+}
+
+var (
+	writeMutex sync.Mutex
+	db         *sql.DB
+	tables     = []string{"baremetal", "cluster", "config", "document", "image", "phase", "secret"}
+)
+
+const (
+	// the table structure used for the records
+	tableCreate = `CREATE TABLE IF NOT EXISTS table (
+		subcomponent varchar(64) null,
+		user varchar(64) null,
+		success tinyint(1) default 0,
+		started timestamp,
+		elapsed timestamp,
+		stopped timestamp)`
+	// the prepared statement used for inserts
+	// TODO (aschiefe): determine if we need to batch inserts
+	insert = "INSERT INTO table(subcomponent, user, success, started, elapsed, stopped) values(?,?,?,?,?,?)"
+)
+
+// Init will create the database if it doesn't exist or open the existing database
+func Init() {
+	intitTables := false
+	// TODO (aschiefe): pull the db location out to the confing
+	if _, err := os.Stat("./sqlite/statistics.db"); os.IsNotExist(err) {
+		intitTables = true
+	}
+	// need to define error so that the program well set the global db variable
+	var err error
+	// TODO (aschiefe): encrypt & password protect the database
+	// TODO (aschiefe): pull the db location out to the confing
+	db, err = sql.Open("sqlite3", "./sqlite/statistics.db")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if intitTables {
+		createTables()
+	}
+}
+
+// createTables is only used when there is no database to write the correct structure for the records
+func createTables() {
+	for index := range tables {
+		stmt, err := db.Prepare(strings.ReplaceAll(tableCreate, "table", tables[index]))
+
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, err = stmt.Exec()
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Tracef("%s table created.", tables[index])
+	}
+}
+
+// NewTransaction establishes the transaction which will record
+func NewTransaction(request configs.WsMessage, user *string) *Transaction {
+	return &Transaction{
+		Table:        request.Component,
+		SubComponent: request.SubComponent,
+		Started:      time.Now().UnixNano() / 1000000,
+		User:         user,
+	}
+}
+
+// Complete will put an entry into the statistics database for the transaction
+func (transaction *Transaction) Complete(errorMessagePresent bool) {
+	if transaction.User != nil && transaction.isRecordable() {
+		stmt, err := db.Prepare(strings.ReplaceAll(insert, "table", string(transaction.Table)))
+		if err != nil {
+			log.Error(err)
+			return
+		}
+
+		started := transaction.Started
+		stopped := time.Now().UnixNano() / 1000000
+
+		success := 0
+		if errorMessagePresent {
+			success = 1
+		}
+
+		writeMutex.Lock()
+		defer writeMutex.Unlock()
+		result, err := stmt.Exec(transaction.SubComponent, transaction.User, success, started, (stopped - started), stopped)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+
+		rows, err := result.RowsAffected()
+		if err != nil {
+			log.Error(err)
+			return
+		}
+
+		log.Tracef("%d rows inserted into %s.", rows, transaction.Table)
+	}
+}
+
+// isRecordable will shuffle through the transaction and determine if we should write it to the database
+func (transaction *Transaction) isRecordable() bool {
+	recordable := true
+	if transaction.Table == configs.Auth {
+		recordable = false
+	}
+	switch transaction.SubComponent {
+	case configs.GetTarget:
+		recordable = false
+	case configs.GetPhaseTree:
+		recordable = false
+	}
+	return recordable
+}

pkg/webservice/auth.go

@@ -44,15 +44,17 @@ func handleAuth(request configs.WsMessage) configs.WsMessage {
 			var token *string
 			authRequest := request.Authentication
 			token, err = createToken(authRequest.ID, authRequest.Password)
+			if token != nil {
 				sessions[request.SessionID].jwt = *token
 				response.SubComponent = configs.Approved
 				response.Token = token
+			}
 		} else {
 			err = errors.New("No AuthRequest found in the request")
 		}
 	case configs.Validate:
 		if request.Token != nil {
-			err = validateToken(*request.Token)
+			_, err = validateToken(*request.Token)
 			response.SubComponent = configs.Approved
 			response.Token = request.Token
 		} else {
@@ -72,7 +74,7 @@ func handleAuth(request configs.WsMessage) configs.WsMessage {
 }
 
 // validate JWT (JSON Web Token)
-func validateToken(tokenString string) error {
+func validateToken(tokenString string) (*string, error) {
 	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
 		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
 			return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
@@ -81,17 +83,20 @@ func validateToken(tokenString string) error {
 	})
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	if _, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
+	if claim, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
-		return nil
+		if user, ok := claim["username"].(string); ok {
+			return &user, nil
 		}
-	return errors.New("Invalid JWT Token")
+		return nil, errors.New("Invalid JWT User")
+	}
+
+	return nil, errors.New("Invalid JWT Token")
 }
 
 // create a JWT (JSON Web Token)
-// TODO (aschiefe): for demo purposes, this is not to be used in production
 func createToken(id string, passwd string) (*string, error) {
 	origPasswdHash, ok := configs.UIConfig.Users[id]
 	if !ok {

pkg/webservice/websocket.go

@@ -25,11 +25,12 @@ import (
 	"github.com/gorilla/websocket"
 	"opendev.org/airship/airshipui/pkg/configs"
 	"opendev.org/airship/airshipui/pkg/log"
+	"opendev.org/airship/airshipui/pkg/statistics"
 )
 
 // Session is a struct to hold information about a given session
 type session struct {
-	id         string
+	sessionID  string
 	jwt        string
 	writeMutex sync.Mutex
 	ws         *websocket.Conn
@@ -74,7 +75,7 @@ func onOpen(response http.ResponseWriter, request *http.Request) {
 	}
 
 	session := newSession(wsConn)
-	log.Debugf("WebSocket session %s established with %s\n", session.id, session.ws.RemoteAddr().String())
+	log.Debugf("WebSocket session %s established with %s\n", session.sessionID, session.ws.RemoteAddr().String())
 
 	go session.onMessage()
 }
@@ -96,9 +97,10 @@ func (session *session) onMessage() {
 		go func() {
 			// test the auth token for request validity on non auth requests
 			// TODO (aschiefe): this will need to be amended when refresh tokens are implemented
+			var user *string
 			if request.Type != configs.UI && request.Component != configs.Auth && request.SubComponent != configs.Authenticate {
 				if request.Token != nil {
-					err = validateToken(*request.Token)
+					user, err = validateToken(*request.Token)
 				} else {
 					err = errors.New("No authentication token found")
 				}
@@ -115,6 +117,10 @@ func (session *session) onMessage() {
 					session.onError(err)
 				}
 			} else {
+				// This is the middleware to be able to record when a transaction starts and ends for the statistics recorder
+				// It is possible for the backend to send messages without a valid user
+				transaction := statistics.NewTransaction(request, user)
+
 				// look through the function map to find the type to handle the request
 				if reqType, ok := functionMap[request.Type]; ok {
 					// the function map may have a component (function) to process the request
@@ -123,12 +129,14 @@ func (session *session) onMessage() {
 						if err = session.webSocketSend(response); err != nil {
 							session.onError(err)
 						}
+						go transaction.Complete(len(response.Error) == 0)
 					} else {
 						if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested component: %s, not found",
 							request.Component), request)); err != nil {
 							session.onError(err)
 						}
 						log.Errorf("Requested component: %s, not found\n", request.Component)
+						go transaction.Complete(false)
 					}
 				} else {
 					if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested type: %s, not found",
@@ -136,6 +144,7 @@ func (session *session) onMessage() {
 						session.onError(err)
 					}
 					log.Errorf("Requested type: %s, not found\n", request.Type)
+					go transaction.Complete(false)
 				}
 			}
 		}()
@@ -144,9 +153,9 @@ func (session *session) onMessage() {
 
 // common websocket close with logging
 func (session *session) onClose() {
-	log.Debugf("Closing websocket for session %s", session.id)
+	log.Debugf("Closing websocket for session %s", session.sessionID)
 	session.ws.Close()
-	delete(sessions, session.id)
+	delete(sessions, session.sessionID)
 }
 
 // common websocket error handling with logging
@@ -176,7 +185,7 @@ func newSession(ws *websocket.Conn) *session {
 	id := uuid.New().String()
 
 	session := &session{
-		id: id,
+		sessionID: id,
 		ws:        ws,
 	}
 
@@ -194,7 +203,7 @@ func (session *session) webSocketSend(response configs.WsMessage) error {
 	session.writeMutex.Lock()
 	defer session.writeMutex.Unlock()
 	response.Timestamp = time.Now().UnixNano() / 1000000
-	response.SessionID = session.id
+	response.SessionID = session.sessionID
 
 	return session.ws.WriteJSON(response)
 }
@@ -216,7 +225,7 @@ func (session *session) sendInit() {
 		Dashboards: configs.UIConfig.Dashboards,
 		AuthMethod: configs.UIConfig.AuthMethod,
 	}); err != nil {
-		log.Errorf("Error receiving / sending init to session %s: %s\n", session.id, err)
+		log.Errorf("Error receiving / sending init to session %s: %s\n", session.sessionID, err)
 	}
 }