From bb428ede66c7f36afbfb547659828c9d3cdb44dc Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Fri, 27 Aug 2021 16:58:51 +0200 Subject: [PATCH 1/9] [#571] Fix Postgres 10 compatibility issue --- backend/server/database/model/event.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/server/database/model/event.go b/backend/server/database/model/event.go index cff0a13bc..47ce53a40 100644 --- a/backend/server/database/model/event.go +++ b/backend/server/database/model/event.go @@ -67,18 +67,18 @@ func GetEventsByPage(db *pg.DB, offset int64, limit int64, level int64, daemonTy q = q.Where("level >= ?", level) } if daemonType != nil { - q = q.Join("JOIN daemon ON daemon.id = CAST (relations->'DaemonID' AS INTEGER)") + q = q.Join("JOIN daemon ON daemon.id = CAST (relations->>'DaemonID' AS INTEGER)") q = q.Where("daemon.name = ?", daemonType) } if appType != nil { - q = q.Join("JOIN app ON app.id = CAST (relations->'AppID' AS INTEGER)") + q = q.Join("JOIN app ON app.id = CAST (relations->>'AppID' AS INTEGER)") q = q.Where("app.type = ?", appType) } if machineID != nil { - q = q.Where("CAST (relations->'MachineID' AS INTEGER) = ?", *machineID) + q = q.Where("CAST (relations->>'MachineID' AS INTEGER) = ?", *machineID) } if userID != nil { - q = q.Where("CAST (relations->'UserID' AS INTEGER) = ?", *userID) + q = q.Where("CAST (relations->>'UserID' AS INTEGER) = ?", *userID) } // prepare sorting expression, offset and limit -- GitLab From 73ad974367352d9bdf74451ae5f31552e47fa006 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Mon, 30 Aug 2021 12:24:01 +0200 Subject: [PATCH 2/9] [#571] Check the database version on start --- backend/server/database/connection.go | 45 +++++++++++++++++++ .../server/database/test/connection_test.go | 18 ++++++++ 2 files changed, 63 insertions(+) diff --git a/backend/server/database/connection.go b/backend/server/database/connection.go index 2fe23c8ac..1bfb6372e 100644 --- a/backend/server/database/connection.go +++ b/backend/server/database/connection.go @@ -15,6 +15,29 @@ import ( type DBLogger struct{} +const ( + minSupportedDatabaseServerVersionMajor = 10 + minSupportedDatabaseServerVersionMinor = 0 + minSupportedDatabaseServerVersionPatch = 0 + minSupportedDatabaseServerVersion = minSupportedDatabaseServerVersionMajor*10000 + + minSupportedDatabaseServerVersionMinor*100 + + minSupportedDatabaseServerVersionPatch +) + +var errUnsupportedDatabaseServerVersion = errors.New("unsupported database version") + +func UnsupportedDatabaseServerVersionError(version int) error { + patch := version % 100 + minor := (version / 100) % 100 + major := version / (100 * 100) + return fmt.Errorf("%w: got %d.%d.%d, required at least %d.%d.%d", + errUnsupportedDatabaseServerVersion, major, minor, patch, + minSupportedDatabaseServerVersionMajor, + minSupportedDatabaseServerVersionMinor, + minSupportedDatabaseServerVersionPatch, + ) +} + // Hook run before SQL query execution. func (d DBLogger) BeforeQuery(c context.Context, q *pg.QueryEvent) (context.Context, error) { // When making queries on the system_user table we want to make sure that @@ -77,6 +100,18 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { if err != nil { return nil, pkgerrors.Wrapf(err, "unable to connect to the database using provided credentials") } + + // Check that a database version is supported + version, err := GetDatabaseServerVersion(db) + if err != nil { + return nil, err + } + + if version < minSupportedDatabaseServerVersion { + err = UnsupportedDatabaseServerVersionError(version) + return nil, err + } + return db, nil } @@ -153,3 +188,13 @@ func Transaction(dbIface interface{}) (tx *pg.Tx, rollback func(), commit func() } return tx, rollback, commit, err } + +// Fetch the connected Postgres version in numeric format. +func GetDatabaseServerVersion(db *PgDB) (int, error) { + var version int + _, err := db.QueryOne(pg.Scan(&version), "SELECT CAST(current_setting('server_version_num') AS integer)") + if err != nil { + return 0, err + } + return version, nil +} diff --git a/backend/server/database/test/connection_test.go b/backend/server/database/test/connection_test.go index 9af5703f2..855bb1cc3 100644 --- a/backend/server/database/test/connection_test.go +++ b/backend/server/database/test/connection_test.go @@ -43,3 +43,21 @@ func TestTransaction(t *testing.T) { // Those two pointers should point at the same object. require.Same(t, tx, tx2) } + +// Tests the logic that fetches database server version. +func TestGetDatabaseServerVersion(t *testing.T) { + db, _, teardown := SetupDatabaseTestCase(t) + defer teardown() + + version, err := dbops.GetDatabaseServerVersion(db) + + require.NoError(t, err) + require.GreaterOrEqual(t, version, 130000) +} + +// Tests the logic that creates an error when the database server has an unsupported version. +func TestCreateUnsupportedDatabaseServerVersionError(t *testing.T) { + err := dbops.UnsupportedDatabaseServerVersionError(90120) + require.EqualValues(t, err.Error(), + "unsupported database version: got 9.1.20, required at least 10.0.0") +} -- GitLab From 7c6511a8ce1ae21c833741be385e2ef196a145f5 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Mon, 30 Aug 2021 12:24:41 +0200 Subject: [PATCH 3/9] [#571] Support new Chrome version --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 0349deda2..d82eedcad 100644 --- a/Rakefile +++ b/Rakefile @@ -955,6 +955,8 @@ if ENV['CHROME_BIN'] CHROME_DRV_VERSION = '87.0.4280.20' elsif out.include? '90.' CHROME_DRV_VERSION = '90.0.4430.72' + elsif out.include? '92.' + CHROME_DRV_VERSION = '92.0.4515.159' else CHROME_DRV_VERSION = "" puts "Cannot match Chrome browser version and chromedriver version" -- GitLab From d39f1069f4b595c6b27d9688d1e653230b6da3f3 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Mon, 30 Aug 2021 12:42:42 +0200 Subject: [PATCH 4/9] [#571] Adjust values in unit test --- backend/server/database/test/connection_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/server/database/test/connection_test.go b/backend/server/database/test/connection_test.go index 855bb1cc3..f8671ee50 100644 --- a/backend/server/database/test/connection_test.go +++ b/backend/server/database/test/connection_test.go @@ -52,7 +52,8 @@ func TestGetDatabaseServerVersion(t *testing.T) { version, err := dbops.GetDatabaseServerVersion(db) require.NoError(t, err) - require.GreaterOrEqual(t, version, 130000) + require.GreaterOrEqual(t, version, 100000) + require.Less(t, version, 200000) } // Tests the logic that creates an error when the database server has an unsupported version. -- GitLab From 0977feb57a4e1fc619fc950d0c41466242b32485 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Mon, 30 Aug 2021 17:28:30 +0200 Subject: [PATCH 5/9] [#571] Avoid to use global vars --- backend/server/database/connection.go | 44 ++++++++----------- .../server/database/test/connection_test.go | 7 --- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/backend/server/database/connection.go b/backend/server/database/connection.go index 1bfb6372e..028b2dbf5 100644 --- a/backend/server/database/connection.go +++ b/backend/server/database/connection.go @@ -15,29 +15,6 @@ import ( type DBLogger struct{} -const ( - minSupportedDatabaseServerVersionMajor = 10 - minSupportedDatabaseServerVersionMinor = 0 - minSupportedDatabaseServerVersionPatch = 0 - minSupportedDatabaseServerVersion = minSupportedDatabaseServerVersionMajor*10000 + - minSupportedDatabaseServerVersionMinor*100 + - minSupportedDatabaseServerVersionPatch -) - -var errUnsupportedDatabaseServerVersion = errors.New("unsupported database version") - -func UnsupportedDatabaseServerVersionError(version int) error { - patch := version % 100 - minor := (version / 100) % 100 - major := version / (100 * 100) - return fmt.Errorf("%w: got %d.%d.%d, required at least %d.%d.%d", - errUnsupportedDatabaseServerVersion, major, minor, patch, - minSupportedDatabaseServerVersionMajor, - minSupportedDatabaseServerVersionMinor, - minSupportedDatabaseServerVersionPatch, - ) -} - // Hook run before SQL query execution. func (d DBLogger) BeforeQuery(c context.Context, q *pg.QueryEvent) (context.Context, error) { // When making queries on the system_user table we want to make sure that @@ -107,9 +84,24 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { return nil, err } - if version < minSupportedDatabaseServerVersion { - err = UnsupportedDatabaseServerVersionError(version) - return nil, err + // Minimal supported database server version + minMajnor := 10 + minMinor := 0 + minPatch := 0 + + minVersion := minMajnor*100*100 + minMinor*100 + minPatch + + if version < minVersion { + currentPatch := version % 100 + currentMinor := (version / 100) % 100 + currentMajnor := version / (100 * 100) + + return nil, pkgerrors.Errorf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d", + currentMajnor, currentMinor, currentPatch, + minMajnor, + minMinor, + minPatch, + ) } return db, nil diff --git a/backend/server/database/test/connection_test.go b/backend/server/database/test/connection_test.go index f8671ee50..b38be0575 100644 --- a/backend/server/database/test/connection_test.go +++ b/backend/server/database/test/connection_test.go @@ -55,10 +55,3 @@ func TestGetDatabaseServerVersion(t *testing.T) { require.GreaterOrEqual(t, version, 100000) require.Less(t, version, 200000) } - -// Tests the logic that creates an error when the database server has an unsupported version. -func TestCreateUnsupportedDatabaseServerVersionError(t *testing.T) { - err := dbops.UnsupportedDatabaseServerVersionError(90120) - require.EqualValues(t, err.Error(), - "unsupported database version: got 9.1.20, required at least 10.0.0") -} -- GitLab From 2b0b981f93f4ae046f6f5975f7e81179184de00f Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Tue, 31 Aug 2021 10:28:57 +0200 Subject: [PATCH 6/9] [#571] Update changelog --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 0b6558c8d..c8b329e92 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1099,4 +1099,4 @@ LEGEND * [perf] Performance related change. * [ui] User Interface change. -*: Backward incompatible or operational change. +*: Backward incompatible or operational change. \ No newline at end of file -- GitLab From d1dae29ff5396369e340835820803b4f5d6af8a6 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Wed, 1 Sep 2021 16:08:39 +0200 Subject: [PATCH 7/9] [#571] Warning on the old databases --- backend/server/database/connection.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/server/database/connection.go b/backend/server/database/connection.go index 028b2dbf5..142e75bc8 100644 --- a/backend/server/database/connection.go +++ b/backend/server/database/connection.go @@ -96,7 +96,8 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { currentMinor := (version / 100) % 100 currentMajnor := version / (100 * 100) - return nil, pkgerrors.Errorf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d", + log.Warnf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d,\n"+ + "Stork was tested using Postgres 11, you use older version on your own risk", currentMajnor, currentMinor, currentPatch, minMajnor, minMinor, -- GitLab From 1d1a091f938f9d6d02659a4fe91a126dc23e8ded Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Thu, 2 Sep 2021 12:17:42 +0200 Subject: [PATCH 8/9] [#571] Reworded some sentences --- ChangeLog.md | 7 +++++++ backend/server/database/connection.go | 4 ++-- doc/install.rst | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index c8b329e92..08456115e 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,10 @@ +* 153 [bug] slawek + + Corrected an issue with fetching Stork events from the + databases running on PostgreSQL 10. Also, the Stork + server requires PostgreSQL version 10 or later. + (Gitlab #571) + * 152 [build] marcin Resolved an issue with building Stork packages in Docker on the MacOS. diff --git a/backend/server/database/connection.go b/backend/server/database/connection.go index 142e75bc8..1b0840045 100644 --- a/backend/server/database/connection.go +++ b/backend/server/database/connection.go @@ -96,8 +96,8 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { currentMinor := (version / 100) % 100 currentMajnor := version / (100 * 100) - log.Warnf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d,\n"+ - "Stork was tested using Postgres 11, you use older version on your own risk", + log.Warnf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d, "+ + "please consider upgrading Postgres server because Stork may not work correctly with this version", currentMajnor, currentMinor, currentPatch, minMajnor, minMinor, diff --git a/doc/install.rst b/doc/install.rst index a833e235b..384273d5b 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -47,8 +47,9 @@ statistics. Stork uses Go implementation for handling TLS connections, certificates and keys. The secrets are stored in the PostgreSQL database, in the `secret` table. -For the ``Stork Server``, a PostgreSQL database (https://www.postgresql.org/) version 11 or later is required. It may work with -PostgreSQL 10, but this has not been tested. The general installation procedure for PostgreSQL is OS-specific and is not included +For the ``Stork Server``, a PostgreSQL database (https://www.postgresql.org/) version 10 +or later is required. Stork will attempt to run with older versions but may not work +correctly. The general installation procedure for PostgreSQL is OS-specific and is not included here. However, please note that Stork uses pgcrypto extensions, which often come in a separate package. For example, a postgresql-crypto package is required on Fedora and postgresql12-contrib is needed on RHEL and CentOS. -- GitLab From 691541f1aadf4339bdf22e0a8380283aff878263 Mon Sep 17 00:00:00 2001 From: Slawek Figiel Date: Thu, 2 Sep 2021 16:07:32 +0200 Subject: [PATCH 9/9] [#571] Supported vesion as global constants --- backend/server/database/connection.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/backend/server/database/connection.go b/backend/server/database/connection.go index 1b0840045..787ad9022 100644 --- a/backend/server/database/connection.go +++ b/backend/server/database/connection.go @@ -13,6 +13,13 @@ import ( log "github.com/sirupsen/logrus" ) +// Minimal supported database Postgres server version. +const ( + minSupportedDatabaseServerVersionMajnor = 10 + minSupportedDatabaseServerVersionMinor = 0 + minSupportedDatabaseServerVersionPatch = 0 +) + type DBLogger struct{} // Hook run before SQL query execution. @@ -84,12 +91,9 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { return nil, err } - // Minimal supported database server version - minMajnor := 10 - minMinor := 0 - minPatch := 0 - - minVersion := minMajnor*100*100 + minMinor*100 + minPatch + minVersion := minSupportedDatabaseServerVersionMajnor*100*100 + + minSupportedDatabaseServerVersionMinor*100 + + minSupportedDatabaseServerVersionPatch if version < minVersion { currentPatch := version % 100 @@ -99,9 +103,9 @@ func NewPgDBConn(pgParams *pg.Options, tracing bool) (*PgDB, error) { log.Warnf("unsupported database server version: got %d.%d.%d, required at least %d.%d.%d, "+ "please consider upgrading Postgres server because Stork may not work correctly with this version", currentMajnor, currentMinor, currentPatch, - minMajnor, - minMinor, - minPatch, + minSupportedDatabaseServerVersionMajnor, + minSupportedDatabaseServerVersionMinor, + minSupportedDatabaseServerVersionPatch, ) } -- GitLab