[1008-move-database-schema-version-check-to-database-libraries] Moved getVersion codes to databases

src/hooks/dhcp/mysql_cb/mysql_cb_impl.cc

@@ -44,19 +44,20 @@ ScopedAuditRevision::~ScopedAuditRevision() {
 MySqlConfigBackendImpl::
 MySqlConfigBackendImpl(const DatabaseConnection::ParameterMap& parameters)
     : conn_(parameters), audit_revision_created_(false) {
-    // Open the database.
-    conn_.openDatabase();
-
-    // Test schema version before we try to prepare statements.
+    // Test schema version first.
     std::pair<uint32_t, uint32_t> code_version(MYSQL_SCHEMA_VERSION_MAJOR,
                                                MYSQL_SCHEMA_VERSION_MINOR);
-/*    std::pair<uint32_t, uint32_t> db_version = getVersion();
+    std::pair<uint32_t, uint32_t> db_version =
+        MySqlConnection::getVersion(parameters);
     if (code_version != db_version) {
         isc_throw(DbOpenError, "MySQL schema version mismatch: need version: "
                   << code_version.first << "." << code_version.second
                   << " found version:  " << db_version.first << "."
                   << db_version.second);
-    } */
+    }
+
+    // Open the database.
+    conn_.openDatabase();
 
     // Enable autocommit. In case transaction is explicitly used, this
     // setting will be overwritten for the transaction. However, there are

src/lib/cql/cql_connection.cc

@@ -75,6 +75,21 @@ CqlConnection::~CqlConnection() {
     }
 }
 
+std::pair<uint32_t, uint32_t>
+CqlConnection::getVersion(const ParameterMap& parameters) {
+    // Get a connection.
+    CqlConnection conn(parameters);
+
+    // Open the database.
+    conn.openDatabase();
+
+    // Prepare statement.
+    conn.prepareStatements(CqlVersionExchange::tagged_statements_);
+
+    std::unique_ptr<CqlVersionExchange> version_exchange(new CqlVersionExchange());
+    return version_exchange->retrieveVersion(conn);
+}
+
 CassConsistency CqlConnection::parseConsistency(std::string value) {
     static std::map<std::string, CassConsistency> consistency_map {
         {"any", CASS_CONSISTENCY_ANY},

src/lib/cql/cql_connection.h

@@ -127,6 +127,19 @@ public:
     /// @brief Destructor
     virtual ~CqlConnection();
 
+    /// @brief Get the schema version.
+    ///
+    /// @param parameters A data structure relating keywords and values
+    ///        concerned with the database.
+    ///
+    /// @return Version number as a pair of unsigned integers.  "first" is the
+    ///         major version number, "second" the minor number.
+    ///
+    /// @throw isc::db::DbOperationError An operation on the open database has
+    ///        failed.
+    static std::pair<uint32_t, uint32_t>
+    getVersion(const ParameterMap& parameters);
+
     /// @brief Prepare statements
     ///
     /// Creates the prepared statements for all of the CQL statements used

src/lib/dhcpsrv/cql_host_data_source.cc

@@ -2320,6 +2320,9 @@ protected:
                             HostPtr& target_host) const;
 
 private:
+    /// @brief Parameters
+    db::DatabaseConnection::ParameterMap parameters_;
+
     /// @brief CQL connection
     mutable CqlConnection dbconn_;
 };  // class CqlHostDataSourceImpl
@@ -2366,14 +2369,8 @@ operator==(const HostKey& key1, const HostKey& key2) {
 }
 
 CqlHostDataSourceImpl::CqlHostDataSourceImpl(const CqlConnection::ParameterMap& parameters)
-    : dbconn_(parameters) {
-    // Open the database.
-    dbconn_.openDatabase();
-
-    // Prepare the version exchange first.
-    dbconn_.prepareStatements(CqlVersionExchange::tagged_statements_);
-
-    // Validate the schema version.
+    : parameters_(parameters), dbconn_(parameters) {
+    // Validate the schema version first.
     std::pair<uint32_t, uint32_t> code_version(CQL_SCHEMA_VERSION_MAJOR,
                                                CQL_SCHEMA_VERSION_MINOR);
     std::pair<uint32_t, uint32_t> db_version = getVersion();
@@ -2384,6 +2381,9 @@ CqlHostDataSourceImpl::CqlHostDataSourceImpl(const CqlConnection::ParameterMap&
                   << db_version.second);
     }
 
+    // Open the database.
+    dbconn_.openDatabase();
+
     // Prepare all possible statements.
     dbconn_.prepareStatements(CqlHostExchange::tagged_statements_);
 }
@@ -2748,8 +2748,7 @@ CqlHostDataSourceImpl::getName() const {
 
 VersionPair
 CqlHostDataSourceImpl::getVersion() const {
-    std::unique_ptr<CqlVersionExchange> version_exchange(new CqlVersionExchange());
-    return (version_exchange->retrieveVersion(dbconn_));
+    return CqlConnection::getVersion(parameters_);
 }
 
 bool

src/lib/dhcpsrv/cql_lease_mgr.cc

@@ -2029,13 +2029,8 @@ CqlLeaseStatsQuery::executeSelect(const CqlConnection& connection, const AnyArra
 }
 
 CqlLeaseMgr::CqlLeaseMgr(const DatabaseConnection::ParameterMap &parameters)
-    : LeaseMgr(), dbconn_(parameters) {
-    dbconn_.openDatabase();
-
-    // Prepare the version exchange first.
-    dbconn_.prepareStatements(CqlVersionExchange::tagged_statements_);
-
-    // Validate the schema version.
+  : LeaseMgr(), parameters_(parameters), dbconn_(parameters) {
+    // Validate the schema version first.
     std::pair<uint32_t, uint32_t> code_version(CQL_SCHEMA_VERSION_MAJOR,
                                                CQL_SCHEMA_VERSION_MINOR);
     std::pair<uint32_t, uint32_t> db_version = getVersion();
@@ -2046,6 +2041,9 @@ CqlLeaseMgr::CqlLeaseMgr(const DatabaseConnection::ParameterMap &parameters)
                   << db_version.second);
     }
 
+    // Open the database.
+    dbconn_.openDatabase();
+
     // Now prepare the rest of the exchanges.
     dbconn_.prepareStatements(CqlLease4Exchange::tagged_statements_);
     dbconn_.prepareStatements(CqlLease6Exchange::tagged_statements_);
@@ -2744,8 +2742,7 @@ VersionPair
 CqlLeaseMgr::getVersion() const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_CQL_GET_VERSION);
 
-    std::unique_ptr<CqlVersionExchange> version_exchange(new CqlVersionExchange());
-    return version_exchange->retrieveVersion(dbconn_);
+    return CqlConnection::getVersion(parameters_);
 }
 
 void

src/lib/dhcpsrv/cql_lease_mgr.h

@@ -578,6 +578,9 @@ public:
     virtual void rollback() override;
 
 private:
+    /// @brief Connection parameters
+    db::DatabaseConnection::ParameterMap parameters_;
+
     /// @brief Database connection object
     mutable db::CqlConnection dbconn_;
 };

src/lib/dhcpsrv/mysql_host_data_source.cc

@@ -1948,11 +1948,9 @@ public:
 
     /// @brief Returns backend version.
     ///
-    /// The method is called by the constructor after opening the database
-    /// but prior to preparing SQL statements, to verify that the schema version
-    /// is correct. Thus it must not rely on a pre-prepared statement or
-    /// formal statement execution error checking.
-    //
+    /// The method is called by the constructor before opening the database
+    /// to verify that the schema version is correct.
+    ///
     /// @return Version number stored in the database, as a pair of unsigned
     ///         integers. "first" is the major version number, "second" the
     ///         minor number.
@@ -2097,6 +2095,9 @@ public:
     /// or dhcp6_options table.
     boost::shared_ptr<MySqlOptionExchange> host_option_exchange_;
 
+    /// @brief The parameters
+    db::DatabaseConnection::ParameterMap parameters_;
+
     /// @brief MySQL connection
     MySqlConnection conn_;
 
@@ -2481,13 +2482,11 @@ MySqlHostDataSourceImpl(const MySqlConnection::ParameterMap& parameters)
                                                      DHCP4_AND_DHCP6)),
       host_ipv6_reservation_exchange_(new MySqlIPv6ReservationExchange()),
       host_option_exchange_(new MySqlOptionExchange()),
+      parameters_(parameters),
       conn_(parameters),
       is_readonly_(false) {
 
-    // Open the database.
-    conn_.openDatabase();
-
-    // Test schema version before we try to prepare statements.
+    // Test schema version first.
     std::pair<uint32_t, uint32_t> code_version(MYSQL_SCHEMA_VERSION_MAJOR,
                                                MYSQL_SCHEMA_VERSION_MINOR);
     std::pair<uint32_t, uint32_t> db_version = getVersion();
@@ -2498,6 +2497,9 @@ MySqlHostDataSourceImpl(const MySqlConnection::ParameterMap& parameters)
                   << db_version.second);
     }
 
+    // Open the database.
+    conn_.openDatabase();
+
     // Enable autocommit. In case transaction is explicitly used, this
     // setting will be overwritten for the transaction. However, there are
     // cases when lack of autocommit could cause transactions to hang
@@ -2551,62 +2553,9 @@ MySqlHostDataSourceImpl::getVersion() const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
               DHCPSRV_MYSQL_HOST_DB_GET_VERSION);
 
-    // Allocate a new statement.
-    MYSQL_STMT *stmt = mysql_stmt_init(conn_.mysql_);
-    if (stmt == NULL) {
-        isc_throw(DbOperationError, "unable to allocate MySQL prepared "
-                  "statement structure, reason: " << mysql_error(conn_.mysql_));
-    }
-
-    // Prepare the statement from SQL text.
-    const char* version_sql = "SELECT version, minor FROM schema_version";
-    int status = mysql_stmt_prepare(stmt, version_sql, strlen(version_sql));
-    if (status != 0) {
-        isc_throw(DbOperationError, "unable to prepare MySQL statement <"
-                  << version_sql << ">, reason: " << mysql_errno(conn_.mysql_));
-    }
-
-    // Execute the prepared statement.
-    if (mysql_stmt_execute(stmt) != 0) {
-        isc_throw(DbOperationError, "cannot execute schema version query <"
-                  << version_sql << ">, reason: " << mysql_errno(conn_.mysql_));
-    }
-
-    // Bind the output of the statement to the appropriate variables.
-    MYSQL_BIND bind[2];
-    memset(bind, 0, sizeof(bind));
-
-    uint32_t major;
-    bind[0].buffer_type = MYSQL_TYPE_LONG;
-    bind[0].is_unsigned = 1;
-    bind[0].buffer = &major;
-    bind[0].buffer_length = sizeof(major);
-
-    uint32_t minor;
-    bind[1].buffer_type = MYSQL_TYPE_LONG;
-    bind[1].is_unsigned = 1;
-    bind[1].buffer = &minor;
-    bind[1].buffer_length = sizeof(minor);
-
-    if (mysql_stmt_bind_result(stmt, bind)) {
-        isc_throw(DbOperationError, "unable to bind result set for <"
-                  << version_sql << ">, reason: " << mysql_errno(conn_.mysql_));
-    }
-
-    // Fetch the data.
-    if (mysql_stmt_fetch(stmt)) {
-        mysql_stmt_close(stmt);
-        isc_throw(DbOperationError, "unable to bind result set for <"
-                  << version_sql << ">, reason: " << mysql_errno(conn_.mysql_));
-    }
-
-    // Discard the statement and its resources
-    mysql_stmt_close(stmt);
-
-    return (std::make_pair(major, minor));
+    return (MySqlConnection::getVersion(parameters_));
 }
 
-
 void
 MySqlHostDataSourceImpl::addStatement(StatementIndex stindex,
                                       std::vector<MYSQL_BIND>& bind) {

src/lib/dhcpsrv/mysql_lease_mgr.cc

@@ -2961,65 +2961,7 @@ MySqlLeaseMgr::getVersion() const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
               DHCPSRV_MYSQL_GET_VERSION);
 
-    // Get a connection.
-    MySqlConnection conn(parameters_);
-
-    // Open the database.
-    conn.openDatabase();
-
-    // Allocate a new statement.
-    MYSQL_STMT *stmt = mysql_stmt_init(conn.mysql_);
-    if (stmt == NULL) {
-        isc_throw(DbOperationError, "unable to allocate MySQL prepared "
-                "statement structure, reason: " << mysql_error(conn.mysql_));
-    }
-
-    // Prepare the statement from SQL text.
-    const char* version_sql = "SELECT version, minor FROM schema_version";
-    int status = mysql_stmt_prepare(stmt, version_sql, strlen(version_sql));
-    if (status != 0) {
-        isc_throw(DbOperationError, "unable to prepare MySQL statement <"
-                  << version_sql << ">, reason: " << mysql_error(conn.mysql_));
-    }
-
-    // Execute the prepared statement.
-    if (mysql_stmt_execute(stmt) != 0) {
-        isc_throw(DbOperationError, "cannot execute schema version query <"
-                  << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
-    }
-
-    // Bind the output of the statement to the appropriate variables.
-    MYSQL_BIND bind[2];
-    memset(bind, 0, sizeof(bind));
-
-    uint32_t major;
-    bind[0].buffer_type = MYSQL_TYPE_LONG;
-    bind[0].is_unsigned = 1;
-    bind[0].buffer = &major;
-    bind[0].buffer_length = sizeof(major);
-
-    uint32_t minor;
-    bind[1].buffer_type = MYSQL_TYPE_LONG;
-    bind[1].is_unsigned = 1;
-    bind[1].buffer = &minor;
-    bind[1].buffer_length = sizeof(minor);
-
-    if (mysql_stmt_bind_result(stmt, bind)) {
-        isc_throw(DbOperationError, "unable to bind result set for <"
-                << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
-    }
-
-    // Fetch the data.
-    if (mysql_stmt_fetch(stmt)) {
-        mysql_stmt_close(stmt);
-        isc_throw(DbOperationError, "unable to bind result set for <"
-                << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
-    }
-
-    // Discard the statement and its resources
-    mysql_stmt_close(stmt);
-
-    return (std::make_pair(major, minor));
+    return (MySqlConnection::getVersion(parameters_));
 }
 
 void

src/lib/dhcpsrv/pgsql_lease_mgr.cc

@@ -1115,9 +1115,8 @@ protected:
 
 PgSqlLeaseMgr::PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters)
     : LeaseMgr(), exchange4_(new PgSqlLease4Exchange()),
-    exchange6_(new PgSqlLease6Exchange()), conn_(parameters) {
-    conn_.openDatabase();
-
+      exchange6_(new PgSqlLease6Exchange()), parameters_(parameters),
+      conn_(parameters) {
     // Validate schema version first.
     std::pair<uint32_t, uint32_t> code_version(PG_SCHEMA_VERSION_MAJOR,
                                                PG_SCHEMA_VERSION_MINOR);
@@ -1130,6 +1129,9 @@ PgSqlLeaseMgr::PgSqlLeaseMgr(const DatabaseConnection::ParameterMap& parameters)
                       << db_version.second);
     }
 
+    // Open the database.
+    conn_.openDatabase();
+
     // Now prepare the SQL statements.
     int i = 0;
     for( ; tagged_statements[i].text != NULL ; ++i) {
@@ -1929,25 +1931,7 @@ PgSqlLeaseMgr::getVersion() const {
     LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL,
               DHCPSRV_PGSQL_GET_VERSION);
 
-    const char* version_sql =  "SELECT version, minor FROM schema_version;";
-    PgSqlResult r(PQexec(conn_, version_sql));
-    if(PQresultStatus(r) != PGRES_TUPLES_OK) {
-        isc_throw(DbOperationError, "unable to execute PostgreSQL statement <"
-                  << version_sql << ", reason: " << PQerrorMessage(conn_));
-    }
-
-    istringstream tmp;
-    uint32_t version;
-    tmp.str(PQgetvalue(r, 0, 0));
-    tmp >> version;
-    tmp.str("");
-    tmp.clear();
-
-    uint32_t minor;
-    tmp.str(PQgetvalue(r, 0, 1));
-    tmp >> minor;
-
-    return (make_pair(version, minor));
+    return (PgSqlConnection::getVersion(parameters_));
 }
 
 void

src/lib/dhcpsrv/pgsql_lease_mgr.h

@@ -770,6 +770,9 @@ private:
     boost::scoped_ptr<PgSqlLease4Exchange> exchange4_; ///< Exchange object
     boost::scoped_ptr<PgSqlLease6Exchange> exchange6_; ///< Exchange object
 
+    /// The parameters.
+    db::DatabaseConnection::ParameterMap parameters_;
+
     /// PostgreSQL connection handle
     db::PgSqlConnection conn_;
 };

src/lib/mysql/mysql_connection.cc

@@ -212,6 +212,70 @@ MySqlConnection::openDatabase() {
     }
 }
 
+// Get schema version.
+
+std::pair<uint32_t, uint32_t>
+MySqlConnection::getVersion(const ParameterMap& parameters) {
+    // Get a connection.
+    MySqlConnection conn(parameters);
+
+    // Open the database.
+    conn.openDatabase();
+
+    // Allocate a new statement.
+    MYSQL_STMT *stmt = mysql_stmt_init(conn.mysql_);
+    if (stmt == NULL) {
+        isc_throw(DbOperationError, "unable to allocate MySQL prepared "
+                "statement structure, reason: " << mysql_error(conn.mysql_));
+    }
+
+    // Prepare the statement from SQL text.
+    const char* version_sql = "SELECT version, minor FROM schema_version";
+    int status = mysql_stmt_prepare(stmt, version_sql, strlen(version_sql));
+    if (status != 0) {
+        isc_throw(DbOperationError, "unable to prepare MySQL statement <"
+                  << version_sql << ">, reason: " << mysql_error(conn.mysql_));
+    }
+
+    // Execute the prepared statement.
+    if (mysql_stmt_execute(stmt) != 0) {
+        isc_throw(DbOperationError, "cannot execute schema version query <"
+                  << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
+    }
+
+    // Bind the output of the statement to the appropriate variables.
+    MYSQL_BIND bind[2];
+    memset(bind, 0, sizeof(bind));
+
+    uint32_t major;
+    bind[0].buffer_type = MYSQL_TYPE_LONG;
+    bind[0].is_unsigned = 1;
+    bind[0].buffer = &major;
+    bind[0].buffer_length = sizeof(major);
+
+    uint32_t minor;
+    bind[1].buffer_type = MYSQL_TYPE_LONG;
+    bind[1].is_unsigned = 1;
+    bind[1].buffer = &minor;
+    bind[1].buffer_length = sizeof(minor);
+
+    if (mysql_stmt_bind_result(stmt, bind)) {
+        isc_throw(DbOperationError, "unable to bind result set for <"
+                << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
+    }
+
+    // Fetch the data.
+    if (mysql_stmt_fetch(stmt)) {
+        mysql_stmt_close(stmt);
+        isc_throw(DbOperationError, "unable to bind result set for <"
+                << version_sql << ">, reason: " << mysql_errno(conn.mysql_));
+    }
+
+    // Discard the statement and its resources
+    mysql_stmt_close(stmt);
+
+    return (std::make_pair(major, minor));
+}
 
 // Prepared statement setup.  The textual form of an SQL statement is stored
 // in a vector of strings (text_statements_) and is used in the output of

src/lib/mysql/mysql_connection.h

@@ -210,6 +210,19 @@ public:
     /// @brief Destructor
     virtual ~MySqlConnection();
 
+    /// @brief Get the schema version.
+    ///
+    /// @param parameters A data structure relating keywords and values
+    ///        concerned with the database.
+    ///
+    /// @return Version number as a pair of unsigned integers.  "first" is the
+    ///         major version number, "second" the minor number.
+    ///
+    /// @throw isc::db::DbOperationError An operation on the open database has
+    ///        failed.
+    static std::pair<uint32_t, uint32_t>
+    getVersion(const ParameterMap& parameters);
+
     /// @brief Prepare Single Statement
     ///
     /// Creates a prepared statement from the text given and adds it to the

src/lib/mysql/tests/mysql_connection_unittest.cc

@@ -390,4 +390,29 @@ TEST_F(MySqlConnectionTest, deleteByValue) {
     EXPECT_TRUE(deleted);
 }
 
+/// @brief Test fixture class for @c MySqlConnection class methods.
+class MySqlSchemaTest : public ::testing::Test {
+public:
+    /// @brief Constructor.
+    MySqlSchemaTest() {
+        // Ensure we have the proper schema.
+        createMySQLSchema();
+    }
+
+    /// @brief Destructor.
+    virtual ~MySqlSchemaTest() {
+        destroyMySQLSchema();
+    }
+};
+
+/// @brief Check that getVersion() returns the expected version.
+TEST_F(MySqlSchemaTest, checkVersion) {
+    // Check version
+    auto parameters = DatabaseConnection::parse(validMySQLConnectionString());
+    std::pair<uint32_t, uint32_t> version;
+    ASSERT_NO_THROW(version = MySqlConnection::getVersion(parameters));
+    EXPECT_EQ(MYSQL_SCHEMA_VERSION_MAJOR, version.first);
+    EXPECT_EQ(MYSQL_SCHEMA_VERSION_MINOR, version.second);
+}
+
 }

src/lib/pgsql/pgsql_connection.cc

@@ -25,6 +25,8 @@
 #define PGSQL_STATECODE_LEN 5
 #include <utils/errcodes.h>
 
+#include <sstream>
+
 using namespace std;
 
 namespace isc {
@@ -127,6 +129,35 @@ PgSqlConnection::~PgSqlConnection() {
     }
 }
 
+std::pair<uint32_t, uint32_t>
+PgSqlConnection::getVersion(const ParameterMap& parameters) {
+    // Get a connection.
+    PgSqlConnection conn(parameters);
+
+    // Open the database.
+    conn.openDatabase();
+
+    const char* version_sql =  "SELECT version, minor FROM schema_version;";
+    PgSqlResult r(PQexec(conn.conn_, version_sql));
+    if (PQresultStatus(r) != PGRES_TUPLES_OK) {
+        isc_throw(DbOperationError, "unable to execute PostgreSQL statement <"
+                  << version_sql << ", reason: " << PQerrorMessage(conn.conn_));
+    }
+
+    istringstream tmp;
+    uint32_t version;
+    tmp.str(PQgetvalue(r, 0, 0));
+    tmp >> version;
+    tmp.str("");
+    tmp.clear();
+
+    uint32_t minor;
+    tmp.str(PQgetvalue(r, 0, 1));
+    tmp >> minor;
+
+    return (make_pair(version, minor));
+}
+
 void
 PgSqlConnection::prepareStatement(const PgSqlTaggedStatement& statement) {
     // Prepare all statements queries with all known fields datatype

src/lib/pgsql/pgsql_connection.h

@@ -311,6 +311,19 @@ public:
     /// @brief Destructor
     virtual ~PgSqlConnection();
 
+    /// @brief Get the schema version.
+    ///
+    /// @param parameters A data structure relating keywords and values
+    ///        concerned with the database.
+    ///
+    /// @return Version number as a pair of unsigned integers.  "first" is the
+    ///         major version number, "second" the minor number.
+    ///
+    /// @throw isc::db::DbOperationError An operation on the open database has
+    ///        failed.
+    static std::pair<uint32_t, uint32_t>
+    getVersion(const ParameterMap& parameters);
+
     /// @brief Prepare Single Statement
     ///
     /// Creates a prepared statement from the text given and adds it to the
@@ -318,7 +331,7 @@ public:
     ///
     /// @param statement SQL statement to be prepared.
     ///
-    /// @throw isc::dhcp::DbOperationError An operation on the open database has
+    /// @throw isc::db::DbOperationError An operation on the open database has
     ///        failed.
     void prepareStatement(const PgSqlTaggedStatement& statement);
 
@@ -332,7 +345,7 @@ public:
     /// @param end_statement Pointer to the statement marking end of the
     /// range of statements to be compiled. This last statement is not compiled.
     ///
-    /// @throw isc::dhcp::DbOperationError An operation on the open database has
+    /// @throw isc::db::DbOperationError An operation on the open database has
     ///        failed.