Add sqlite for statistics / auditing for each transaction

This allows for a built in audit database for user actions
We can also see how often commands are run, how long they take
as well as who and when they're run

TODO: use sqlcipher to encrypt at rest & password protect the db

Change-Id: Ic7c8927bcfdd46ede3fe6a5aca4f57892ca3f3d4
This commit is contained in:
Schiefelbein, Andrew 2020-09-29 13:46:21 -05:00
parent e4d36d3c54
commit b4583b1db5
9 changed files with 218 additions and 20 deletions

3
.gitignore vendored
View File

@ -15,6 +15,9 @@ dist
etc/*.pem etc/*.pem
etc/*.json etc/*.json
# sqlite database files
sqlite/*.db
# Only exists if Bazel was run # Only exists if Bazel was run
/bazel-out /bazel-out

View File

@ -12,7 +12,7 @@ Clone the Airship UI repository and build.
make # Note running behind a proxy can cause issues, notes on solving is in the Appendix make # Note running behind a proxy can cause issues, notes on solving is in the Appendix
**NOTE:** Make will install node.js-v12.16.3 into your tools directory and will use that as the node binary for the UI **NOTE:** Make will install node.js-v12.16.3 into your tools directory and will use that as the node binary for the UI
building, testing and linting. For windows this can be done using [cygwin](https://www.cygwin.com/) make. building, testing and linting. For windows this can be done using [cygwin](https://www.cygwin.com/) make. Windows may also require [tdm-gcc](https://jmeubank.github.io/tdm-gcc/) for the sqlite dependency.
Run the airshipui binary Run the airshipui binary
@ -135,6 +135,28 @@ it:
export NODE_EXTRA_CA_CERTS=/<path>/<truststore>.pem export NODE_EXTRA_CA_CERTS=/<path>/<truststore>.pem
## Issues with SQLITE on Windows
You may experience issues when attempting to install SQLITE:
```
C:\<path>\sqlite> go get github.com/mattn/go-sqlite3
# github.com/mattn/go-sqlite3
/usr/lib/gcc/x86_64-pc-cygwin/10/../../../../x86_64-pc-cygwin/bin/ld: cannot find -lmingwex
/usr/lib/gcc/x86_64-pc-cygwin/10/../../../../x86_64-pc-cygwin/bin/ld: cannot find -lmingw32
collect2: error: ld returned 1 exit status
go: failed to remove work dir: GetFileInformationByHandle C:\Users\someUser\AppData\Local\Temp\go-build323470906\NUL: Incorrect function.
```
To fix this you will need to install [tdm-gcc](https://jmeubank.github.io/tdm-gcc/) and set your path to reference the tdm-gcc first on the path:
```
C:\<path>\sqlite> set PATH=c:\TDM-GCC-64\bin;%PATH%
```
Test that the tdm-gcc is first on the path
```
C:\<path>\sqlite> which gcc
/cygdrive/c/TDM-GCC-64/bin/gcc
```
You should be able to sucessfully run a 'go get github.com/mattn/go-sqlite3' without error
### Optional proxy settings ### Optional proxy settings
#### Environment settings for wget or curl #### Environment settings for wget or curl

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/google/uuid v1.1.1 github.com/google/uuid v1.1.1
github.com/gorilla/websocket v1.4.2 github.com/gorilla/websocket v1.4.2
github.com/mattn/go-sqlite3 v1.14.3
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.0.0 github.com/spf13/cobra v1.0.0
github.com/stretchr/testify v1.6.1 github.com/stretchr/testify v1.6.1

2
go.sum
View File

@ -706,6 +706,8 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=

View File

@ -24,6 +24,7 @@ import (
"opendev.org/airship/airshipui/pkg/configs" "opendev.org/airship/airshipui/pkg/configs"
"opendev.org/airship/airshipui/pkg/ctl" "opendev.org/airship/airshipui/pkg/ctl"
"opendev.org/airship/airshipui/pkg/log" "opendev.org/airship/airshipui/pkg/log"
"opendev.org/airship/airshipui/pkg/statistics"
"opendev.org/airship/airshipui/pkg/webservice" "opendev.org/airship/airshipui/pkg/webservice"
) )
@ -69,6 +70,9 @@ func launch(cmd *cobra.Command, args []string) {
log.Fatalf("config %s", err) log.Fatalf("config %s", err)
} }
// Start the statistics database
statistics.Init()
// allows for the circular reference to the webservice package to be broken and allow for the sending // allows for the circular reference to the webservice package to be broken and allow for the sending
// of arbitrary messages from any package to the websocket // of arbitrary messages from any package to the websocket
ctl.Init() ctl.Init()

152
pkg/statistics/recorder.go Executable file
View File

@ -0,0 +1,152 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package statistics
import (
"database/sql"
"os"
"strings"
"sync"
"time"
_ "github.com/mattn/go-sqlite3" // this is required for the sqlite driver
"opendev.org/airship/airshipui/pkg/configs"
"opendev.org/airship/airshipui/pkg/log"
)
// Transaction will record the details of the CTL transaction and record them to the DB
type Transaction struct {
Table configs.WsComponentType
SubComponent configs.WsSubComponentType
User *string
Started int64
}
var (
writeMutex sync.Mutex
db *sql.DB
tables = []string{"baremetal", "cluster", "config", "document", "image", "phase", "secret"}
)
const (
// the table structure used for the records
tableCreate = `CREATE TABLE IF NOT EXISTS table (
subcomponent varchar(64) null,
user varchar(64) null,
success tinyint(1) default 0,
started timestamp,
elapsed timestamp,
stopped timestamp)`
// the prepared statement used for inserts
// TODO (aschiefe): determine if we need to batch inserts
insert = "INSERT INTO table(subcomponent, user, success, started, elapsed, stopped) values(?,?,?,?,?,?)"
)
// Init will create the database if it doesn't exist or open the existing database
func Init() {
intitTables := false
// TODO (aschiefe): pull the db location out to the confing
if _, err := os.Stat("./sqlite/statistics.db"); os.IsNotExist(err) {
intitTables = true
}
// need to define error so that the program well set the global db variable
var err error
// TODO (aschiefe): encrypt & password protect the database
// TODO (aschiefe): pull the db location out to the confing
db, err = sql.Open("sqlite3", "./sqlite/statistics.db")
if err != nil {
log.Fatal(err)
}
if intitTables {
createTables()
}
}
// createTables is only used when there is no database to write the correct structure for the records
func createTables() {
for index := range tables {
stmt, err := db.Prepare(strings.ReplaceAll(tableCreate, "table", tables[index]))
if err != nil {
log.Fatal(err)
}
_, err = stmt.Exec()
if err != nil {
log.Fatal(err)
}
log.Tracef("%s table created.", tables[index])
}
}
// NewTransaction establishes the transaction which will record
func NewTransaction(request configs.WsMessage, user *string) *Transaction {
return &Transaction{
Table: request.Component,
SubComponent: request.SubComponent,
Started: time.Now().UnixNano() / 1000000,
User: user,
}
}
// Complete will put an entry into the statistics database for the transaction
func (transaction *Transaction) Complete(errorMessagePresent bool) {
if transaction.User != nil && transaction.isRecordable() {
stmt, err := db.Prepare(strings.ReplaceAll(insert, "table", string(transaction.Table)))
if err != nil {
log.Error(err)
return
}
started := transaction.Started
stopped := time.Now().UnixNano() / 1000000
success := 0
if errorMessagePresent {
success = 1
}
writeMutex.Lock()
defer writeMutex.Unlock()
result, err := stmt.Exec(transaction.SubComponent, transaction.User, success, started, (stopped - started), stopped)
if err != nil {
log.Error(err)
return
}
rows, err := result.RowsAffected()
if err != nil {
log.Error(err)
return
}
log.Tracef("%d rows inserted into %s.", rows, transaction.Table)
}
}
// isRecordable will shuffle through the transaction and determine if we should write it to the database
func (transaction *Transaction) isRecordable() bool {
recordable := true
if transaction.Table == configs.Auth {
recordable = false
}
switch transaction.SubComponent {
case configs.GetTarget:
recordable = false
case configs.GetPhaseTree:
recordable = false
}
return recordable
}

View File

@ -44,15 +44,17 @@ func handleAuth(request configs.WsMessage) configs.WsMessage {
var token *string var token *string
authRequest := request.Authentication authRequest := request.Authentication
token, err = createToken(authRequest.ID, authRequest.Password) token, err = createToken(authRequest.ID, authRequest.Password)
sessions[request.SessionID].jwt = *token if token != nil {
response.SubComponent = configs.Approved sessions[request.SessionID].jwt = *token
response.Token = token response.SubComponent = configs.Approved
response.Token = token
}
} else { } else {
err = errors.New("No AuthRequest found in the request") err = errors.New("No AuthRequest found in the request")
} }
case configs.Validate: case configs.Validate:
if request.Token != nil { if request.Token != nil {
err = validateToken(*request.Token) _, err = validateToken(*request.Token)
response.SubComponent = configs.Approved response.SubComponent = configs.Approved
response.Token = request.Token response.Token = request.Token
} else { } else {
@ -72,7 +74,7 @@ func handleAuth(request configs.WsMessage) configs.WsMessage {
} }
// validate JWT (JSON Web Token) // validate JWT (JSON Web Token)
func validateToken(tokenString string) error { func validateToken(tokenString string) (*string, error) {
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
@ -81,17 +83,20 @@ func validateToken(tokenString string) error {
}) })
if err != nil { if err != nil {
return err return nil, err
} }
if _, ok := token.Claims.(jwt.MapClaims); ok && token.Valid { if claim, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
return nil if user, ok := claim["username"].(string); ok {
return &user, nil
}
return nil, errors.New("Invalid JWT User")
} }
return errors.New("Invalid JWT Token")
return nil, errors.New("Invalid JWT Token")
} }
// create a JWT (JSON Web Token) // create a JWT (JSON Web Token)
// TODO (aschiefe): for demo purposes, this is not to be used in production
func createToken(id string, passwd string) (*string, error) { func createToken(id string, passwd string) (*string, error) {
origPasswdHash, ok := configs.UIConfig.Users[id] origPasswdHash, ok := configs.UIConfig.Users[id]
if !ok { if !ok {

View File

@ -25,11 +25,12 @@ import (
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"opendev.org/airship/airshipui/pkg/configs" "opendev.org/airship/airshipui/pkg/configs"
"opendev.org/airship/airshipui/pkg/log" "opendev.org/airship/airshipui/pkg/log"
"opendev.org/airship/airshipui/pkg/statistics"
) )
// Session is a struct to hold information about a given session // Session is a struct to hold information about a given session
type session struct { type session struct {
id string sessionID string
jwt string jwt string
writeMutex sync.Mutex writeMutex sync.Mutex
ws *websocket.Conn ws *websocket.Conn
@ -74,7 +75,7 @@ func onOpen(response http.ResponseWriter, request *http.Request) {
} }
session := newSession(wsConn) session := newSession(wsConn)
log.Debugf("WebSocket session %s established with %s\n", session.id, session.ws.RemoteAddr().String()) log.Debugf("WebSocket session %s established with %s\n", session.sessionID, session.ws.RemoteAddr().String())
go session.onMessage() go session.onMessage()
} }
@ -96,9 +97,10 @@ func (session *session) onMessage() {
go func() { go func() {
// test the auth token for request validity on non auth requests // test the auth token for request validity on non auth requests
// TODO (aschiefe): this will need to be amended when refresh tokens are implemented // TODO (aschiefe): this will need to be amended when refresh tokens are implemented
var user *string
if request.Type != configs.UI && request.Component != configs.Auth && request.SubComponent != configs.Authenticate { if request.Type != configs.UI && request.Component != configs.Auth && request.SubComponent != configs.Authenticate {
if request.Token != nil { if request.Token != nil {
err = validateToken(*request.Token) user, err = validateToken(*request.Token)
} else { } else {
err = errors.New("No authentication token found") err = errors.New("No authentication token found")
} }
@ -115,6 +117,10 @@ func (session *session) onMessage() {
session.onError(err) session.onError(err)
} }
} else { } else {
// This is the middleware to be able to record when a transaction starts and ends for the statistics recorder
// It is possible for the backend to send messages without a valid user
transaction := statistics.NewTransaction(request, user)
// look through the function map to find the type to handle the request // look through the function map to find the type to handle the request
if reqType, ok := functionMap[request.Type]; ok { if reqType, ok := functionMap[request.Type]; ok {
// the function map may have a component (function) to process the request // the function map may have a component (function) to process the request
@ -123,12 +129,14 @@ func (session *session) onMessage() {
if err = session.webSocketSend(response); err != nil { if err = session.webSocketSend(response); err != nil {
session.onError(err) session.onError(err)
} }
go transaction.Complete(len(response.Error) == 0)
} else { } else {
if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested component: %s, not found", if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested component: %s, not found",
request.Component), request)); err != nil { request.Component), request)); err != nil {
session.onError(err) session.onError(err)
} }
log.Errorf("Requested component: %s, not found\n", request.Component) log.Errorf("Requested component: %s, not found\n", request.Component)
go transaction.Complete(false)
} }
} else { } else {
if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested type: %s, not found", if err = session.webSocketSend(requestErrorHelper(fmt.Sprintf("Requested type: %s, not found",
@ -136,6 +144,7 @@ func (session *session) onMessage() {
session.onError(err) session.onError(err)
} }
log.Errorf("Requested type: %s, not found\n", request.Type) log.Errorf("Requested type: %s, not found\n", request.Type)
go transaction.Complete(false)
} }
} }
}() }()
@ -144,9 +153,9 @@ func (session *session) onMessage() {
// common websocket close with logging // common websocket close with logging
func (session *session) onClose() { func (session *session) onClose() {
log.Debugf("Closing websocket for session %s", session.id) log.Debugf("Closing websocket for session %s", session.sessionID)
session.ws.Close() session.ws.Close()
delete(sessions, session.id) delete(sessions, session.sessionID)
} }
// common websocket error handling with logging // common websocket error handling with logging
@ -176,8 +185,8 @@ func newSession(ws *websocket.Conn) *session {
id := uuid.New().String() id := uuid.New().String()
session := &session{ session := &session{
id: id, sessionID: id,
ws: ws, ws: ws,
} }
// keep track of the session // keep track of the session
@ -194,7 +203,7 @@ func (session *session) webSocketSend(response configs.WsMessage) error {
session.writeMutex.Lock() session.writeMutex.Lock()
defer session.writeMutex.Unlock() defer session.writeMutex.Unlock()
response.Timestamp = time.Now().UnixNano() / 1000000 response.Timestamp = time.Now().UnixNano() / 1000000
response.SessionID = session.id response.SessionID = session.sessionID
return session.ws.WriteJSON(response) return session.ws.WriteJSON(response)
} }
@ -216,7 +225,7 @@ func (session *session) sendInit() {
Dashboards: configs.UIConfig.Dashboards, Dashboards: configs.UIConfig.Dashboards,
AuthMethod: configs.UIConfig.AuthMethod, AuthMethod: configs.UIConfig.AuthMethod,
}); err != nil { }); err != nil {
log.Errorf("Error receiving / sending init to session %s: %s\n", session.id, err) log.Errorf("Error receiving / sending init to session %s: %s\n", session.sessionID, err)
} }
} }

0
sqlite/.gitkeep Normal file
View File