mirror/musikcube
1
0
Fork 0
mirror of https://github.com/clangen/musikcube.git synced 2025-03-29 19:20:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

A couple database optimizations that should make the indexing process faster.

Browse Source
This commit is contained in:
Casey Langen 2016-07-28 19:28:51 -07:00
parent 5d9c58fd16
commit 5a290aa6d5
5 changed files with 68 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/core/db/Connection.cpp
View File

@ -39,19 +39,19 @@
#include <boost/thread/thread.hpp>
#include <sqlite/sqlite3.h>
using namespace musik::core::db;
static boost::mutex globalMutex;
boost::mutex Connection::globalMutex;
using namespace musik::core::db;
Connection::Connection()
: connection(nullptr)
, transactionCounter(0) {
this->Maintenance(true);
this->UpdateReferenceCount(true);
}
Connection::~Connection(){
this->Close();
this->Maintenance(false);
this->UpdateReferenceCount(false);
}
//////////////////////////////////////////
@ -209,9 +209,8 @@ int Connection::Execute(const wchar_t* sql) {
return musik::core::db::Okay;
}
void Connection::Analyze(){
boost::mutex::scoped_lock lock(Connection::globalMutex);
// this->Execute("ANALYZE");
void Connection::Checkpoint() {
sqlite3_wal_checkpoint(this->connection, nullptr);
}
//////////////////////////////////////////
@ -237,56 +236,52 @@ int Connection::LastInsertedId(){
///
///This will set all the initial PRAGMAS
//////////////////////////////////////////
void Connection::Initialize(unsigned int cache){
void Connection::Initialize(unsigned int cache) {
// sqlite3_enable_shared_cache(1);
sqlite3_busy_timeout(this->connection,10000);
sqlite3_busy_timeout(this->connection, 10000);
sqlite3_exec(this->connection,"PRAGMA synchronous=OFF",NULL,NULL,NULL); // Not a critical DB. Sync set to OFF
sqlite3_exec(this->connection,"PRAGMA page_size=4096",NULL,NULL,NULL); // According to windows standard page size
sqlite3_exec(this->connection,"PRAGMA auto_vacuum=0",NULL,NULL,NULL); // No autovaccum.
sqlite3_exec(this->connection, "PRAGMA synchronous=OFF", NULL, NULL, NULL); // Not a critical DB. Sync set to OFF
sqlite3_exec(this->connection, "PRAGMA page_size=4096", NULL, NULL, NULL); // According to windows standard page size
sqlite3_exec(this->connection, "PRAGMA auto_vacuum=0", NULL, NULL, NULL); // No autovaccum.
sqlite3_exec(this->connection, "PRAGMA auto_vacuum=0", NULL, NULL, NULL); // No autovaccum.
sqlite3_exec(this->connection, "PRAGMA journal_mode=WAL", NULL, NULL, NULL); // Allow reading while writing (write-ahead-logging)
if(cache!=0){
if (cache != 0) {
// Divide by 4 to since the page_size is 4096
// Total cache is the same as page_size*cache_size
cache = cache/4;
cache = cache / 4;
std::string cacheSize("PRAGMA cache_size=" + boost::lexical_cast<std::string>(cache));
sqlite3_exec(this->connection,cacheSize.c_str(),NULL,NULL,NULL); // size * 1.5kb = 6Mb cache
sqlite3_exec(this->connection,cacheSize.c_str(), NULL, NULL, NULL); // size * 1.5kb = 6Mb cache
}
sqlite3_exec(this->connection,"PRAGMA case_sensitive_like=0",NULL,NULL,NULL); // More speed if case insensitive
sqlite3_exec(this->connection,"PRAGMA count_changes=0",NULL,NULL,NULL); // If set it counts changes on SQL UPDATE. More speed when not.
sqlite3_exec(this->connection,"PRAGMA legacy_file_format=OFF",NULL,NULL,NULL); // No reason to be backwards compatible :)
sqlite3_exec(this->connection,"PRAGMA temp_store=MEMORY",NULL,NULL,NULL); // MEMORY, not file. More speed.
sqlite3_exec(this->connection, "PRAGMA case_sensitive_like=0", NULL, NULL, NULL); // More speed if case insensitive
sqlite3_exec(this->connection, "PRAGMA count_changes=0", NULL, NULL, NULL); // If set it counts changes on SQL UPDATE. More speed when not.
sqlite3_exec(this->connection, "PRAGMA legacy_file_format=OFF", NULL, NULL, NULL); // No reason to be backwards compatible :)
sqlite3_exec(this->connection, "PRAGMA temp_store=MEMORY", NULL, NULL, NULL); // MEMORY, not file. More speed.
}
//////////////////////////////////////////
///\brief
///Interrupts the current running statement(s)
//////////////////////////////////////////
void Connection::Interrupt(){
boost::mutex::scoped_lock lock(this->mutex);
sqlite3_interrupt(this->connection);
}
void Connection::Maintenance(bool init){
void Connection::UpdateReferenceCount(bool init) {
boost::mutex::scoped_lock lock(globalMutex);
// Need to be locked throuout all Connections
boost::mutex::scoped_lock lock(Connection::globalMutex);
static int count = 0;
static int counter(0);
if(init){
if(counter==0){
if (init) {
if (count == 0) {
sqlite3_initialize();
}
++counter;
}else{
--counter;
if(counter==0){
++count;
}
else {
--count;
if (count <= 0){
sqlite3_shutdown();
count = 0;
}
}
}

32
src/core/db/Connection.h
View File

@ -43,54 +43,38 @@
#include <boost/utility.hpp>
#include <boost/thread/mutex.hpp>
//////////////////////////////////////////
// Forward declare
struct sqlite3;
struct sqlite3_stmt;
//////////////////////////////////////////
namespace musik { namespace core { namespace db {
namespace musik{ namespace core{ namespace db{
//////////////////////////////////////////
///\brief
///Database Wrapper
///
///A Connection to the database
//////////////////////////////////////////
class Connection : boost::noncopyable{
class Connection : boost::noncopyable {
public:
Connection();
~Connection();
int Open(const char *database,unsigned int options=0,unsigned int cache=0);
int Open(const std::string &database,unsigned int options=0,unsigned int cache=0);
int Open(const char *database, unsigned int options = 0, unsigned int cache = 0);
int Open(const std::string &database, unsigned int options = 0, unsigned int cache = 0);
int Close();
int Execute(const char* sql);
int Execute(const wchar_t* sql);
int LastInsertedId();
void Interrupt();
void Analyze();
void Checkpoint();
private:
void Initialize(unsigned int cache);
void UpdateReferenceCount(bool init);
int StepStatement(sqlite3_stmt *stmt);
friend class Statement;
friend class ScopedTransaction;
int StepStatement(sqlite3_stmt *stmt);
int transactionCounter;
sqlite3 *connection;
boost::mutex mutex;
static boost::mutex globalMutex;
void Maintenance(bool init);
};
} } }

6
src/core/db/ScopedTransaction.cpp
View File

@ -59,8 +59,11 @@ void ScopedTransaction::CommitAndRestart() {
}
void ScopedTransaction::Begin(){
/* we use an IMMEDIATE transaction because we have write-ahead-logging
enabled on this instance, this generally results in faster queries
and also allows reads while writing */
if (this->connection->transactionCounter == 0) {
this->connection->Execute("BEGIN TRANSACTION");
this->connection->Execute("BEGIN IMMEDIATE TRANSACTION");
}
++this->connection->transactionCounter;
@ -75,6 +78,7 @@ void ScopedTransaction::End() {
}
else {
this->connection->Execute("COMMIT TRANSACTION");
this->connection->Checkpoint();
}
}
}

24
src/core/library/Indexer.cpp
View File

@ -53,9 +53,11 @@
#include <boost/bind.hpp>
#define MULTI_THREADED_INDEXER 1
#define STRESS_TEST_DB 0
static const std::string TAG = "Indexer";
static const int MAX_THREADS = 10;
static const int NOTIFY_INTERVAL = 2000;
using namespace musik::core;
using namespace musik::core::metadata;
@ -293,6 +295,26 @@ void Indexer::ReadMetadataFromFile(
track.SetValue("path_id", pathId.c_str());
track.Save(this->dbConnection, this->libraryPath);
this->filesSaved++;
#ifdef STRESS_TEST_DB
#define INC(track, key, x) \
{ \
std::string val = track.GetValue(key); \
val += (char) ('a' + x); \
track.ClearValue(key); \
track.SetValue(key, val.c_str()); \
}
for (int i = 0; i < 10; i++) {
track.SetId(0);
INC(track, "title", i);
INC(track, "artist", i);
INC(track, "album_artist", i);
INC(track, "album", i);
track.Save(this->dbConnection, this->libraryPath);
this->filesSaved++;
}
#endif
}
}
@ -329,7 +351,7 @@ void Indexer::SyncDirectory(
std::vector<Thread> threads;
for( ; file != end && !this->Exited() && !this->Restarted(); file++) {
if (this->filesSaved.load() > 200) {
if (this->filesSaved.load() > NOTIFY_INTERVAL) {
if (this->trackTransaction) {
this->trackTransaction->CommitAndRestart();
}

2
src/core/library/LocalLibrary.cpp
View File

@ -374,6 +374,4 @@ void LocalLibrary::CreateDatabase(db::Connection &db){
db.Execute("CREATE INDEX IF NOT EXISTS metavalues_index1 ON meta_values (meta_key_id)");
db.Execute("CREATE INDEX IF NOT EXISTS playlist_index ON playlist_tracks (playlist_id,sort_order)");
db.Analyze();
}