rpms/akonadi
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

backport master/ branch commits to test sqlite backend concurrency support

Browse source
This commit is contained in:
Rex Dieter 2014-04-16 09:23:26 -05:00
commit fcd69dcca6
3 changed files with 367 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

190
0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch Normal file
View file

@ -0,0 +1,190 @@
diff --git a/qsqlite/src/qsql_sqlite.cpp b/qsqlite/src/qsql_sqlite.cpp
index c1e9508..5da232f 100644
--- a/qsqlite/src/qsql_sqlite.cpp
+++ b/qsqlite/src/qsql_sqlite.cpp
@@ -528,7 +528,7 @@ static int qGetSqliteOpenMode(QString opts)
return SQLITE_OPEN_READONLY;
}
// The SQLITE_OPEN_NOMUTEX flag causes the database connection to be in the multi-thread mode
- return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX;
+ return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
}
/*
@@ -543,8 +543,10 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c
if (db.isEmpty())
return false;
+ sqlite3_enable_shared_cache(1);
if (sqlite3_open_v2(db.toUtf8().constData(), &d->access, qGetSqliteOpenMode(conOpts), NULL) == SQLITE_OK) {
sqlite3_busy_timeout(d->access, qGetSqliteTimeout(conOpts));
+ sqlite3_extended_result_codes(d->access, 1);
setOpen(true);
setOpenError(false);
return true;
diff --git a/qsqlite/src/sqlite_blocking.cpp b/qsqlite/src/sqlite_blocking.cpp
index c0fe3f2..180685c 100644
--- a/qsqlite/src/sqlite_blocking.cpp
+++ b/qsqlite/src/sqlite_blocking.cpp
@@ -1,63 +1,94 @@
+/*
+ Copyright (c) 2009 Bertjan Broeksema <broeksema@kde.org>
+ Copyright (c) 2014 Daniel Vrátil <dvratil@redhat.com>
+
+ This library is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Library General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or (at your
+ option) any later version.
+
+ This library is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
+ License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; see the file COPYING.LIB. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ 02110-1301, USA.
+*/
+
#include "sqlite_blocking.h"
#include <sqlite3.h>
-#ifndef _WIN32
-#include <unistd.h>
-#else
-#include <Windows.h>
-#define usleep(x) Sleep(x/1000)
-#endif
-
-#include "qdebug.h"
+
+#include <QMutex>
+#include <QWaitCondition>
#include "qstringbuilder.h"
#include "qthread.h"
+#include <QDateTime>
+
+/* Based on example in http://www.sqlite.org/unlock_notify.html */
-QString debugString()
+struct UnlockNotification {
+ bool fired;
+ QWaitCondition cond;
+ QMutex mutex;
+};
+
+static void qSqlite3UnlockNotifyCb(void **apArg, int nArg)
{
- return QString( QLatin1Literal("[QSQLITE3: ") + QString::number( quint64( QThread::currentThreadId() ) ) + QLatin1Literal("] ") );
+ for (int i = 0; i < nArg; ++i) {
+ UnlockNotification *ntf = static_cast<UnlockNotification*>(apArg[i]);
+ ntf->mutex.lock();
+ ntf->fired = true;
+ ntf->cond.wakeOne();
+ ntf->mutex.unlock();
+ }
}
-int sqlite3_blocking_step( sqlite3_stmt *pStmt )
+static int qSqlite3WaitForUnlockNotify(sqlite3 *db)
{
- // NOTE: The example at http://www.sqlite.org/unlock_notify.html says to wait
- // for SQLITE_LOCK but for some reason I don't understand I get
- // SQLITE_BUSY.
- int rc = sqlite3_step( pStmt );
-
- QThread::currentThreadId();
- if ( rc == SQLITE_BUSY )
- qDebug() << debugString() << "sqlite3_blocking_step: Entering while loop";
-
- while( rc == SQLITE_BUSY ) {
- usleep(5000);
- sqlite3_reset( pStmt );
- rc = sqlite3_step( pStmt );
-
- if ( rc != SQLITE_BUSY ) {
- qDebug() << debugString() << "sqlite3_blocking_step: Leaving while loop";
+ int rc;
+ UnlockNotification un;
+ un.fired = false;
+
+ rc = sqlite3_unlock_notify(db, qSqlite3UnlockNotifyCb, (void *)&un);
+ Q_ASSERT(rc == SQLITE_LOCKED || rc == SQLITE_OK);
+
+ if (rc == SQLITE_OK) {
+ un.mutex.lock();
+ if (!un.fired) {
+ un.cond.wait(&un.mutex);
}
+ un.mutex.unlock();
}
return rc;
}
-int sqlite3_blocking_prepare16_v2( sqlite3 *db, /* Database handle. */
- const void *zSql, /* SQL statement, UTF-16 encoded */
- int nSql, /* Length of zSql in bytes. */
- sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
- const void **pzTail /* OUT: Pointer to unused portion of zSql */ )
+int sqlite3_blocking_step(sqlite3_stmt *pStmt)
{
- int rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail );
-
- if ( rc == SQLITE_BUSY )
- qDebug() << debugString() << "sqlite3_blocking_prepare16_v2: Entering while loop";
+ int rc;
+ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_step(pStmt))) {
+ rc = qSqlite3WaitForUnlockNotify(sqlite3_db_handle(pStmt));
+ if (rc != SQLITE_OK) {
+ break;
+ }
+ sqlite3_reset(pStmt);
+ }
- while( rc == SQLITE_BUSY ) {
- usleep(500000);
- rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail );
+ return rc;
+}
- if ( rc != SQLITE_BUSY ) {
- qDebug() << debugString() << "sqlite3_prepare16_v2: Leaving while loop";
+int sqlite3_blocking_prepare16_v2(sqlite3 *db, const void *zSql, int nSql,
+ sqlite3_stmt **ppStmt, const void **pzTail)
+{
+ int rc;
+ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_prepare16_v2(db, zSql, nSql, ppStmt, pzTail))) {
+ rc = qSqlite3WaitForUnlockNotify(db);
+ if (rc != SQLITE_OK) {
+ break;
}
}
diff --git a/qsqlite/src/sqlite_blocking.h b/qsqlite/src/sqlite_blocking.h
index 0d6f6a0..9f13946 100644
--- a/qsqlite/src/sqlite_blocking.h
+++ b/qsqlite/src/sqlite_blocking.h
@@ -1,3 +1,22 @@
+/*
+ Copyright (c) 2009 Bertjan Broeksema <broeksema@kde.org>
+
+ This library is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Library General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or (at your
+ option) any later version.
+
+ This library is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
+ License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; see the file COPYING.LIB. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ 02110-1301, USA.
+*/
+
#ifndef SQLITE_BLOCKING_H
#define SQLITE_BLOCKING_H

167
0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch Normal file
View file

@ -0,0 +1,167 @@
From 24413dc44b0637d6c64e6b2105c2bcf1b99849a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dan=20Vr=C3=A1til?= <dvratil@redhat.com>
Date: Sun, 6 Apr 2014 19:50:38 +0200
Subject: [PATCH 13/16] Disable global transaction mutex for QSQLITE3 and
enable transaction recording
Our QSQLITE3 driver now supports concurrency, so we don't need to serialize
transactions in DataStore anymore. It is however still needed for the
QSQLITE driver shipped with Qt.
Secondary, concurrency support also means possible transactions deadlocks and
timeouts, so we also need to enable transaction recording and replaying for
the QSQLITE3 backend.
---
server/src/storage/datastore.cpp | 18 +++++++++++-------
server/src/storage/datastore.h | 2 +-
server/src/storage/dbtype.cpp | 5 +++++
server/src/storage/dbtype.h | 3 +++
server/src/storage/querybuilder.cpp | 20 ++++++++++++++++----
server/src/storage/querybuilder.h | 2 +-
6 files changed, 37 insertions(+), 13 deletions(-)
diff --git a/server/src/storage/datastore.cpp b/server/src/storage/datastore.cpp
index 57d1e4e..0f04fa5 100644
--- a/server/src/storage/datastore.cpp
+++ b/server/src/storage/datastore.cpp
@@ -61,8 +61,8 @@ using namespace Akonadi::Server;
static QMutex sTransactionMutex;
bool DataStore::s_hasForeignKeyConstraints = false;
-#define TRANSACTION_MUTEX_LOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.lock()
-#define TRANSACTION_MUTEX_UNLOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.unlock()
+#define TRANSACTION_MUTEX_LOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.lock()
+#define TRANSACTION_MUTEX_UNLOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.unlock()
/***************************************************************************
* DataStore *
@@ -1083,23 +1083,27 @@ QDateTime DataStore::dateTimeToQDateTime( const QByteArray &dateTime )
void DataStore::addQueryToTransaction( const QSqlQuery &query, bool isBatch )
{
- DbType::Type dbType = DbType::type( m_database );
// This is used for replaying deadlocked transactions, so only record queries
// for backends that support concurrent transactions.
- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) {
+ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) {
return;
}
m_transactionQueries.append( qMakePair( query, isBatch ) );
}
-QSqlQuery DataStore::retryLastTransaction()
+QSqlQuery DataStore::retryLastTransaction( bool rollbackFirst )
{
- DbType::Type dbType = DbType::type( m_database );
- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) {
+ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) {
return QSqlQuery();
}
+ if ( rollbackFirst ) {
+ // In some cases the SQL database won't rollback the failed transaction, so
+ // we need to do it manually
+ m_database.driver()->rollbackTransaction();
+ }
+
// The database has rolled back the actual transaction, so reset the counter
// to 0 and start a new one in beginTransaction(). Then restore the level
// because this has to be completely transparent to the original caller
diff --git a/server/src/storage/datastore.h b/server/src/storage/datastore.h
index 8b4a2b7..8a0fe01 100644
--- a/server/src/storage/datastore.h
+++ b/server/src/storage/datastore.h
@@ -317,7 +317,7 @@ protected:
* @return Returns an invalid query when error occurs, or the last replayed
* query on success.
*/
- QSqlQuery retryLastTransaction();
+ QSqlQuery retryLastTransaction( bool rollbackFirst );
private Q_SLOTS:
void sendKeepAliveQuery();
diff --git a/server/src/storage/dbtype.cpp b/server/src/storage/dbtype.cpp
index 495f532..7df2fb1 100644
--- a/server/src/storage/dbtype.cpp
+++ b/server/src/storage/dbtype.cpp
@@ -39,3 +39,8 @@ DbType::Type DbType::typeForDriverName( const QString &driverName )
}
return Unknown;
}
+
+bool DbType::isSystemSQLite( const QSqlDatabase &db )
+{
+ return db.driverName() == QLatin1String( "QSQLITE" );
+}
diff --git a/server/src/storage/dbtype.h b/server/src/storage/dbtype.h
index a95a833..3595604 100644
--- a/server/src/storage/dbtype.h
+++ b/server/src/storage/dbtype.h
@@ -42,6 +42,9 @@ namespace DbType
/** Returns the type for the given driver name. */
Type typeForDriverName( const QString &driverName );
+ /** Returns true when using QSQLITE driver shipped with Qt, FALSE otherwise */
+ bool isSystemSQLite( const QSqlDatabase &db );
+
} // namespace DbType
} // namespace Server
} // namespace Akonadi
diff --git a/server/src/storage/querybuilder.cpp b/server/src/storage/querybuilder.cpp
index 0abad4a..0530b11 100644
--- a/server/src/storage/querybuilder.cpp
+++ b/server/src/storage/querybuilder.cpp
@@ -320,10 +320,10 @@ QString QueryBuilder::buildQuery()
return statement;
}
-bool QueryBuilder::retryLastTransaction()
+bool QueryBuilder::retryLastTransaction( bool rollback )
{
#ifndef QUERYBUILDER_UNITTEST
- mQuery = DataStore::self()->retryLastTransaction();
+ mQuery = DataStore::self()->retryLastTransaction( rollback );
return !mQuery.lastError().isValid();
#else
return true;
@@ -400,9 +400,21 @@ bool QueryBuilder::exec()
akDebug() << mQuery.lastError().text();
return retryLastTransaction();
}
+ } else if ( mDatabaseType == DbType::Sqlite && !DbType::isSystemSQLite( DataStore::self()->database() ) ) {
+ const int error = mQuery.lastError().number();
+ if ( error == 6 /* SQLITE_LOCKED */ ) {
+ akDebug() << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction";
+ akDebug() << mQuery.lastError().text();
+ return retryLastTransaction();
+ } else if ( error == 5 /* SQLITE_BUSY */ ) {
+ akDebug() << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction";
+ akDebug() << mQuery.lastError().text();
+ return retryLastTransaction( true );
+ }
} else if ( mDatabaseType == DbType::Sqlite ) {
- // We can't have a transaction deadlock in SQLite, because it does not support
- // concurrent transactions and DataStore serializes them through a global lock.
+ // We can't have a transaction deadlock in SQLite when using driver shipped
+ // with Qt, because it does not support concurrent transactions and DataStore
+ // serializes them through a global lock.
}
akError() << "DATABASE ERROR:";
diff --git a/server/src/storage/querybuilder.h b/server/src/storage/querybuilder.h
index 235a099..b380f93 100644
--- a/server/src/storage/querybuilder.h
+++ b/server/src/storage/querybuilder.h
@@ -244,7 +244,7 @@ class QueryBuilder
*/
void sqliteAdaptUpdateJoin( Query::Condition &cond );
- bool retryLastTransaction();
+ bool retryLastTransaction( bool rollback = false);
private:
QString mTable;
--
1.9.0

11
akonadi.spec
View file

@ -19,7 +19,7 @@
Summary: PIM Storage Service
Name: akonadi
Version: 1.12.1
Release: 3%{?dist}
Release: 4%{?dist}
License: LGPLv2+
URL: http://community.kde.org/KDE_PIM/Akonadi
@ -38,6 +38,9 @@ Source10: akonadiserverrc.mysql
## upstreamable patches
## upstream patches
# master branch
Patch212: 0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch
Patch213: 0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch
%define mysql_conf_timestamp 20140415
@ -94,6 +97,9 @@ See also: %{_sysconfdir}/akonadi/mysql-global.conf
%prep
%setup -q -n akonadi-%{version}
%patch212 -p1 -b .0012
%patch213 -p1 -b .0013
%build
mkdir -p %{_target_platform}
@ -198,6 +204,9 @@ fi
%changelog
* Wed Apr 16 2014 Rex Dieter <rdieter@fedoraproject.org> 1.12.1-4
- backport master/ branch commits to test sqlite backend concurrency support
* Wed Apr 16 2014 Rex Dieter <rdieter@fedoraproject.org> 1.12.1-3
- WITH_SOPRANO=OFF (kde-4.13,fc21+)