update threading lock to find dead lock trigger

This commit is contained in:
Dario 2019-12-06 15:29:46 +01:00
parent 436765ed63
commit 75038be86a
11 changed files with 181 additions and 68 deletions

View File

@ -28,7 +28,7 @@ Languages PageRequestMessagedHandler::chooseLanguage(Poco::Net::HTTPServerReques
$lang = in_array($lang, $acceptLang) ? $lang : 'en';
*/
std::string accept_languages = request.get("HTTP_ACCEPT_LANGUAGE", "");
printf("accept header: %s\n", accept_languages.data());
printf("[PageRequestMessagedHandler::chooseLanguage] accept header: %s\n", accept_languages.data());
}
}

View File

@ -28,7 +28,14 @@ SessionManager::~SessionManager()
bool SessionManager::init()
{
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::init] exception timout mutex: %s\n", ex.displayText().data());
return false;
}
//mWorkingMutex.lock();
int i;
DISASM_MISALIGN;
for (i = 0; i < VALIDATE_MAX; i++) {
@ -51,14 +58,20 @@ bool SessionManager::init()
mInitalized = true;
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return true;
}
void SessionManager::deinitalize()
{
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::deinitalize] exception timout mutex: %s\n", ex.displayText().data());
return;
}
//mWorkingMutex.lock();
while (mEmptyRequestStack.size()) {
mEmptyRequestStack.pop();
@ -75,7 +88,7 @@ void SessionManager::deinitalize()
mInitalized = false;
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
}
bool SessionManager::isValid(const std::string& subject, SessionValidationTypes validationType)
@ -117,7 +130,14 @@ Session* SessionManager::getNewSession(int* handle)
checkTimeoutSession();
// lock
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::getNewSession] exception timout mutex: %s\n", ex.displayText().data());
return false;
}
//mWorkingMutex.lock();
//UniLib::controller::TaskPtr checkSessionTimeout(new CheckSessionTimeouted);
//checkSessionTimeout->scheduleTask(checkSessionTimeout);
@ -130,7 +150,7 @@ Session* SessionManager::getNewSession(int* handle)
if (resultIt != mRequestSessionMap.end() && !resultIt->second->isActive()) {
Session* result = resultIt->second;
result->reset();
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
if (handle) {
*handle = local_handle;
@ -145,7 +165,7 @@ Session* SessionManager::getNewSession(int* handle)
// check if already exist, if get new
int newHandle = generateNewUnusedHandle();
if (!newHandle) {
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return nullptr;
}
@ -153,7 +173,7 @@ Session* SessionManager::getNewSession(int* handle)
mRequestSessionMap.insert(std::pair<int, Session*>(newHandle, requestSession));
requestSession->setActive(true);
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
if (handle) {
*handle = newHandle;
@ -171,12 +191,19 @@ bool SessionManager::releaseSession(int requestHandleSession)
printf("[SessionManager::%s] not initialized any more\n", __FUNCTION__);
return false;
}
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::releaseSession] exception timout mutex: %s\n", ex.displayText().data());
return false;
}
//mWorkingMutex.lock();
auto it = mRequestSessionMap.find(requestHandleSession);
if (it == mRequestSessionMap.end()) {
//printf("[SessionManager::releaseRequestSession] requestSession with handle: %d not found\n", requestHandleSession);
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return false;
}
Session* session = it->second;
@ -191,7 +218,7 @@ bool SessionManager::releaseSession(int requestHandleSession)
if (!newHandle) {
delete session;
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return true;
}
@ -199,7 +226,7 @@ bool SessionManager::releaseSession(int requestHandleSession)
mRequestSessionMap.insert(std::pair<int, Session*>(newHandle, session));
mEmptyRequestStack.push(newHandle);
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return true;
}
@ -211,7 +238,14 @@ bool SessionManager::isExist(int requestHandleSession)
return false;
}
bool result = false;
mWorkingMutex.lock();
//mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::isExist] exception timout mutex: %s\n", ex.displayText().data());
return false;
}
auto it = mRequestSessionMap.find(requestHandleSession);
if (it != mRequestSessionMap.end()) {
result = true;
@ -219,7 +253,7 @@ bool SessionManager::isExist(int requestHandleSession)
printf("[SessionManager::isExist] session isn't active\n");
}
}
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return result;
}
@ -248,27 +282,41 @@ Session* SessionManager::getSession(int handle)
}
if (0 == handle) return nullptr;
Session* result = nullptr;
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::getSession] exception timout mutex: %s\n", ex.displayText().data());
return result;
}
//mWorkingMutex.lock();
auto it = mRequestSessionMap.find(handle);
if (it != mRequestSessionMap.end()) {
result = it->second;
if (!result->isActive()) {
//printf("[SessionManager::getSession] session isn't active\n");
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return nullptr;
}
//result->setActive(true);
result->updateTimeout();
}
//printf("[SessionManager::getSession] handle: %ld\n", handle);
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return result;
}
Session* SessionManager::findByEmailVerificationCode(long long emailVerificationCode)
{
Session* result = nullptr;
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::findByEmailVerificationCode] exception timout mutex: %s\n", ex.displayText().data());
return result;
}
//mWorkingMutex.lock();
for (auto it = mRequestSessionMap.begin(); it != mRequestSessionMap.end(); it++) {
if (it->second->getEmailVerificationCode() == emailVerificationCode) {
result = it->second;
@ -279,14 +327,22 @@ Session* SessionManager::findByEmailVerificationCode(long long emailVerification
break;
}
}
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
return result;
}
void SessionManager::checkTimeoutSession()
{
mWorkingMutex.lock();
try {
Poco::Mutex::ScopedLock _lock(mWorkingMutex, 500);
}
catch (Poco::TimeoutException &ex) {
printf("[SessionManager::checkTimeoutSession] exception timout mutex: %s\n", ex.displayText().data());
return;
}
//mWorkingMutex.lock();
auto now = Poco::DateTime();
// random variance within 10 seconds for timeout to make it harder to get information and hack the server
auto timeout = Poco::Timespan(ServerConfig::g_SessionTimeout * 60, randombytes_random() % 10000000);
@ -299,7 +355,7 @@ void SessionManager::checkTimeoutSession()
toRemove.push(it->first);
}
}
mWorkingMutex.unlock();
//mWorkingMutex.unlock();
while (toRemove.size() > 0) {
int handle = toRemove.top();

View File

@ -16,6 +16,7 @@
#include "Poco/RegularExpression.h"
#include "Poco/Net/HTTPServerRequest.h"
#include "Poco/Net/HTTPServerResponse.h"
#include "Poco/Mutex.h"
#include <mutex>
#include <map>
@ -66,7 +67,7 @@ public:
void checkTimeoutSession();
// delete all current active login cookies
void deleteLoginCookies(Poco::Net::HTTPServerRequest& request, Poco::Net::HTTPServerResponse& response, Session* activeSession = nullptr);
static void deleteLoginCookies(Poco::Net::HTTPServerRequest& request, Poco::Net::HTTPServerResponse& response, Session* activeSession = nullptr);
protected:
SessionManager();
@ -74,7 +75,7 @@ protected:
int generateNewUnusedHandle();
// access mutex
std::mutex mWorkingMutex;
Poco::Mutex mWorkingMutex;
// sessions storage
std::map<int, Session*> mRequestSessionMap;

View File

@ -0,0 +1,22 @@
#include "MultithreadContainer.h"
#include "ErrorList.h"
namespace UniLib {
namespace lib {
void MultithreadContainer::lock(const char* stackDetails/* = nullptr*/)
{
try {
mWorkMutex.lock(500);
}
catch (Poco::TimeoutException& ex) {
ErrorList errors;
if (stackDetails) {
errors.addError(new Error("MultithreadContainer::lock", stackDetails));
}
errors.addError(new ParamError("MultithreadContainer::lock", "lock timeout", ex.displayText()));
errors.sendErrorsAsEmail();
}
}
}
}

View File

@ -42,7 +42,8 @@ namespace UniLib {
{
public:
inline void lock() { mWorkMutex.lock(); }
void lock(const char* stackDetails = nullptr);
inline void unlock() {mWorkMutex.unlock();}
protected:
private:

View File

@ -98,7 +98,7 @@ Session::~Session()
void Session::reset()
{
//printf("[Session::reset]\n");
lock();
lock("Session::reset");
mSessionUser = nullptr;
@ -117,7 +117,7 @@ void Session::reset()
void Session::updateTimeout()
{
lock();
lock("Session::updateTimeout");
mLastActivity = Poco::DateTime();
unlock();
}
@ -295,7 +295,7 @@ bool Session::updateEmailVerification(Poco::UInt64 emailVerificationCode)
bool Session::startProcessingTransaction(const std::string& proto_message_base64)
{
lock();
lock("Session::startProcessingTransaction");
HASH hs = ProcessingTransaction::calculateHash(proto_message_base64);
// check if it is already running or waiting
for (auto it = mProcessingTransactions.begin(); it != mProcessingTransactions.end(); it++) {
@ -318,7 +318,7 @@ bool Session::startProcessingTransaction(const std::string& proto_message_base64
Poco::AutoPtr<ProcessingTransaction> Session::getNextReadyTransaction(size_t* working/* = nullptr*/)
{
lock();
lock("Session::getNextReadyTransaction");
if (working) {
*working = 0;
}
@ -350,7 +350,7 @@ Poco::AutoPtr<ProcessingTransaction> Session::getNextReadyTransaction(size_t* wo
void Session::finalizeTransaction(bool sign, bool reject)
{
lock();
lock("Session::finalizeTransaction");
if (mCurrentActiveProcessingTransaction.isNull()) {
unlock();
return;
@ -370,7 +370,7 @@ void Session::finalizeTransaction(bool sign, bool reject)
size_t Session::getProcessingTransactionCount()
{
size_t count = 0;
lock();
lock("Session::getProcessingTransactionCount");
for (auto it = mProcessingTransactions.begin(); it != mProcessingTransactions.end(); it++) {
@ -402,7 +402,7 @@ bool Session::isPwdValid(const std::string& pwd)
UserStates Session::loadUser(const std::string& email, const std::string& password)
{
//Profiler usedTime;
lock();
lock("Session::loadUser");
if (mSessionUser && mSessionUser->getEmail() != email) {
mSessionUser = nullptr;
}
@ -437,6 +437,7 @@ UserStates Session::loadUser(const std::string& email, const std::string& passwo
bool Session::deleteUser()
{
lock("Session::deleteUser");
bool bResult = false;
if(mSessionUser) {
bResult = mSessionUser->deleteFromDB();
@ -444,13 +445,13 @@ bool Session::deleteUser()
if(!bResult) {
addError(new Error(gettext("Benutzer"), gettext("Fehler beim L&ouml;schen des Accounts. Bitte logge dich erneut ein und versuche es nochmal.")));
}
unlock();
return bResult;
}
void Session::setLanguage(Languages lang)
{
lock();
lock("Session::setLanguage");
if (mLanguageCatalog.isNull() || mLanguageCatalog->getLanguage() != lang) {
auto lm = LanguageManager::getInstance();
mLanguageCatalog = lm->getFreeCatalog(lang);
@ -461,7 +462,7 @@ void Session::setLanguage(Languages lang)
Languages Session::getLanguage()
{
Languages lang = LANG_NULL;
lock();
lock("Session::getLanguage");
if (!mLanguageCatalog.isNull()) {
lang = mLanguageCatalog->getLanguage();
}
@ -489,6 +490,7 @@ void Session::detectSessionState()
return;
}
UserStates userState = mSessionUser->getUserState();
/*
if (mSessionUser->getDBId() == 0) {
updateState(SESSION_STATE_CRYPTO_KEY_GENERATED);
@ -604,7 +606,7 @@ bool Session::loadFromEmailVerificationCode(Poco::UInt64 emailVerificationCode)
void Session::updateState(SessionStates newState)
{
lock();
lock("Session::updateState");
if (!mActive) return;
updateTimeout();
//printf("[%s] newState: %s\n", __FUNCTION__, translateSessionStateToString(newState));
@ -618,7 +620,7 @@ void Session::updateState(SessionStates newState)
const char* Session::getSessionStateString()
{
SessionStates state;
lock();
lock(" Session::getSessionStateString");
state = mState;
unlock();
return translateSessionStateToString(state);

View File

@ -26,6 +26,7 @@
class WriteEmailVerification;
enum SessionStates {
@ -91,12 +92,12 @@ public:
void updateState(SessionStates newState);
const char* getSessionStateString();
inline SessionStates getSessionState() { SessionStates s; lock(); s = mState; unlock(); return s; }
inline SessionStates getSessionState() { SessionStates s; lock("Session::getSessionState"); s = mState; unlock(); return s; }
inline Poco::UInt64 getEmailVerificationCode() { return mEmailVerificationCode; }
inline bool isActive() { bool bret = false; lock(); bret = mActive; unlock(); return bret; }
inline void setActive(bool active) { lock(); mActive = active; unlock(); }
inline bool isActive() { bool bret = false; lock("Session::isActive"); bret = mActive; unlock(); return bret; }
inline void setActive(bool active) { lock("Sessions::setActive"); mActive = active; unlock(); }
inline Poco::DateTime getLastActivity() { return mLastActivity; }

View File

@ -434,7 +434,7 @@ bool User::validatePassphrase(const std::string& passphrase)
bool User::isEmptyPassword()
{
bool bRet = false;
lock();
lock("User::isEmptyPassword");
//printf("[User::isEmptyPassword] pwd hashed: %d, running: %d, this: %d\n",
// mPasswordHashed, !mCreateCryptoKeyTask.isNull(), this);
bRet = mPasswordHashed == 0 && (mCreateCryptoKeyTask.isNull() || mCreateCryptoKeyTask->isTaskFinished());
@ -445,7 +445,7 @@ bool User::isEmptyPassword()
UserStates User::getUserState()
{
UserStates state;
lock();
lock("User::getUserState");
state = mState;
unlock();
return state;
@ -453,7 +453,7 @@ UserStates User::getUserState()
Poco::JSON::Object User::getJson()
{
lock();
lock("User::getJson");
Poco::JSON::Object userObj;
userObj.set("first_name", mFirstName);
@ -472,19 +472,19 @@ bool User::setNewPassword(const std::string& newPassword)
{
//Profiler timeUsed;
if (newPassword == "") {
lock();
lock("User::setNewPassword");
addError(new Error("Passwort", "Ist leer."));
unlock();
return false;
}
if (!mCreateCryptoKeyTask.isNull() && !mCreateCryptoKeyTask->isTaskFinished()) {
lock();
lock("User::setNewPassword");
addError(new Error("Passwort", "Wird bereits erstellt, bitte in ca. 1 sekunde neuladen."));
unlock();
return false;
}
duplicate();
lock();
lock("User::setNewPassword");
//printf("[User::setNewPassword] start create crypto key task with this: %d\n", this);
mCreateCryptoKeyTask = new UserCreateCryptoKey(this, newPassword, ServerConfig::g_CPUScheduler);
mCreateCryptoKeyTask->scheduleTask(mCreateCryptoKeyTask);
@ -503,7 +503,7 @@ bool User::setNewPassword(const std::string& newPassword)
void User::setEmailChecked()
{
lock();
lock("User::setEmailChecked");
mEmailChecked = true;
if (mState <= USER_EMAIL_NOT_ACTIVATED) {
if (mPublicHex == "") {
@ -535,7 +535,7 @@ bool User::validatePwd(const std::string& pwd, ErrorList* validationErrorsToPrin
return false;
}
crypto_shorthash((unsigned char*)&pwdHashed, *cmpCryptoKey, crypto_box_SEEDBYTES, *ServerConfig::g_ServerCryptoKey);
lock();
lock("User::validatePwd");
if (pwdHashed == mPasswordHashed) {
if (!mCryptoKey) {
mCryptoKey = cmpCryptoKey;
@ -556,7 +556,7 @@ bool User::validatePwd(const std::string& pwd, ErrorList* validationErrorsToPrin
bool User::validateIdentHash(HASH hash)
{
lock();
lock("User::validateIdentHash");
HASH local_hash = DRMakeStringHash(mEmail.data(), mEmail.size());
unlock();
return local_hash == hash;
@ -589,7 +589,7 @@ bool User::deleteFromDB()
}
try {
lock();
lock("User::deleteFromDB");
auto result = deleteFromDB.execute();
unlock();
//printf("[User::deleteFromDB] %s deleted: %d\n", tables[i].data(), result);
@ -608,12 +608,13 @@ bool User::deleteFromDB()
void User::duplicate()
{
mReferenceMutex.lock();
Poco::Mutex::ScopedLock _lock(mReferenceMutex);
//mReferenceMutex.lock();
mReferenceCount++;
#ifdef DEBUG_USER_DELETE_ENV
printf("[User::duplicate] new value: %d\n", mReferenceCount);
#endif
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
}
void User::release()
@ -621,17 +622,18 @@ void User::release()
if (!mCreateCryptoKeyTask.isNull() && mCreateCryptoKeyTask->isTaskFinished()) {
mCreateCryptoKeyTask = nullptr;
}
mReferenceMutex.lock();
Poco::Mutex::ScopedLock _lock(mReferenceMutex);
//mReferenceMutex.lock();
mReferenceCount--;
#ifdef DEBUG_USER_DELETE_ENV
printf("[User::release] new value: %d, this: %d\n", mReferenceCount, this);
#endif
if (0 == mReferenceCount) {
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
delete this;
return;
}
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
}
@ -646,7 +648,7 @@ MemoryBin* User::createCryptoKey(const std::string& password)
sha_context context_sha512;
//unsigned char* hash512 = (unsigned char*)malloc(SHA_512_SIZE);
if (SHA_512_SIZE < crypto_pwhash_SALTBYTES) {
lock();
lock("User::createCryptoKey");
addError(new Error(__FUNCTION__, "sha512 is to small for libsodium pwhash saltbytes"));
unlock();
return nullptr;
@ -666,7 +668,7 @@ MemoryBin* User::createCryptoKey(const std::string& password)
//Bin32Bytes* key = mm->get32Bytes();
if (crypto_pwhash(*key, key->size(), password.data(), password.size(), hash512_salt, 10U, 33554432, 2) != 0) {
lock();
lock("User::createCryptoKey");
addError(new ParamError(__FUNCTION__, " error creating pwd hash, maybe to much memory requestet? error:", strerror(errno)));
unlock();
//printf("[User::%s] error creating pwd hash, maybe to much memory requestet? error: %s\n", __FUNCTION__, strerror(errno));
@ -952,16 +954,30 @@ MemoryBin* User::getPrivKey()
bool User::setPrivKey(const MemoryBin* privKey)
{
if (!hasCryptoKey()) {
lock();
lock("User::setPrivKey");
addError(new Error("User::getPrivKey", "no crypto key set for encrypting priv key"));
unlock();
return false;
}
auto encyrptedPrivKey = encrypt(privKey);
lock();
lock("User::setPrivKey");
mState = USER_COMPLETE;
mPrivateKey = encrypt(privKey);
unlock();
return true;
}
void User::lock(const char* stateInfos/* = nullptr*/)
{
try {
mWorkingMutex.lock(500);
}
catch (Poco::TimeoutException& ex) {
addError(new ParamError("User::lock", "timeout exception", ex.displayText()));
if (stateInfos) {
addError(new ParamError("User::lock", "stateInfos", stateInfos));
}
sendErrorsAsEmail();
}
}

View File

@ -125,7 +125,7 @@ protected:
inline passwordHashed getPwdHashed() { lock(); auto ret = mPasswordHashed; unlock(); return ret; }
inline void setPwdHashed(passwordHashed pwdHashed) { lock(); mPasswordHashed = pwdHashed; unlock(); }
inline void lock() { mWorkingMutex.lock(); }
void lock(const char* stateInfos = nullptr);
inline void unlock() { mWorkingMutex.unlock(); }
MemoryBin* getPrivKey();

View File

@ -1,5 +1,5 @@
#include "Task.h"
#include "../lib/ErrorList.h"
namespace UniLib {
namespace controller {
@ -59,26 +59,40 @@ namespace UniLib {
void Task::duplicate()
{
mReferenceMutex.lock();
Poco::Mutex::ScopedLock _lock(mReferenceMutex);
//mReferenceMutex.lock();
mReferenceCount++;
//printf("[Task::duplicate] new value: %d\n", mReferenceCount);
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
}
void Task::release()
{
mReferenceMutex.lock();
//mReferenceMutex.lock();
Poco::Mutex::ScopedLock _lock(mReferenceMutex);
mReferenceCount--;
//printf("[Task::release] new value: %d\n", mReferenceCount);
if (0 == mReferenceCount) {
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
delete this;
return;
}
mReferenceMutex.unlock();
//mReferenceMutex.unlock();
}
void Task::lock()
{
try {
mWorkingMutex.lock(500);
}
catch (Poco::TimeoutException& ex) {
ErrorList errors;
errors.addError(new ParamError("Task::lock", getResourceType(), ex.displayText()));
errors.sendErrorsAsEmail();
}
}
void Task::setTaskFinished() {
lock();
mFinished = true;

View File

@ -73,7 +73,7 @@ namespace UniLib {
inline void lock() {mWorkingMutex.lock();}
void lock();
inline void unlock() {mWorkingMutex.unlock();}
inline void setParentTaskPtrInArray(TaskPtr task, size_t index)