From f632739b44883a6ed82cb8ab7bd134cbdc6831da Mon Sep 17 00:00:00 2001 From: ruanshudong Date: Tue, 31 Mar 2020 15:55:57 +0800 Subject: [PATCH] fix tarstat, tarsproperty, mac: mem, windows: mmap, linux: shm --- .github/workflows/ccpp-win.yml | 2 +- PropertyServer/PropertyDbManager.cpp | 10 +-- PropertyServer/PropertyHashMap.h | 14 ++-- PropertyServer/PropertyImp.cpp | 9 +-- PropertyServer/PropertyReapThread.cpp | 22 +++---- PropertyServer/PropertyServer.cpp | 30 +++------ StatServer/ReapSSDThread.cpp | 20 +++--- StatServer/StatDbManager.cpp | 10 +-- StatServer/StatHashMap.h | 12 ++-- StatServer/StatImp.cpp | 4 +- StatServer/StatServer.cpp | 65 ++----------------- .../framework/sql/template/tars.tarsproperty | 2 +- deploy/framework/sql/template/tars.tarsstat | 1 - tarscpp | 2 +- 14 files changed, 70 insertions(+), 133 deletions(-) diff --git a/.github/workflows/ccpp-win.yml b/.github/workflows/ccpp-win.yml index 67403271..965edb3d 100644 --- a/.github/workflows/ccpp-win.yml +++ b/.github/workflows/ccpp-win.yml @@ -22,5 +22,5 @@ jobs: - name: configure run: cd build; cmake --version; cmake .. -A X64 - name: make - run: cd build; cmake --build . --config release --target run-all + run: cd build; cmake --build . --config release diff --git a/PropertyServer/PropertyDbManager.cpp b/PropertyServer/PropertyDbManager.cpp index 6cde56d9..c5496138 100644 --- a/PropertyServer/PropertyDbManager.cpp +++ b/PropertyServer/PropertyDbManager.cpp @@ -207,7 +207,7 @@ PropertyDbManager::PropertyDbManager() { vector &vDb = m_iter1->second; - FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl; +// FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl; ++m_iter1; } @@ -380,7 +380,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat TLOGDEBUG("insert " << strTbName << " affected:" << iCount << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; iCount = 0; } @@ -390,7 +390,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat { pMysql->execute(osSql.str()); TLOGDEBUG("insert " << strTbName << " affected:" << iCount << endl); - FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; +// FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; } } catch (TC_Mysql_Exception& ex) @@ -597,8 +597,8 @@ int PropertyDbManager::insert2MultiDbs(int iIndex, const PropertyMsg &propertyms int64_t iEnd = tars::TC_TimeProvider::getInstance()->getNowMs(); TLOGDEBUG("insert|" << iIndex << "|" << getIpAndPort(iIndex) << "|" << sDate << "|" << sFlag << "|" << propertymsg.size() << "|" << (iEnd - iBegin) << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag - << "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag +// << "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl; } } catch(TC_Mysql_Exception& ex) diff --git a/PropertyServer/PropertyHashMap.h b/PropertyServer/PropertyHashMap.h index 5ee94cff..4c263f51 100644 --- a/PropertyServer/PropertyHashMap.h +++ b/PropertyServer/PropertyHashMap.h @@ -27,13 +27,15 @@ using namespace tars; typedef StatPropMsgBody PropBody; typedef StatPropMsgHead PropHead; -typedef TarsHashMap PropHashMap; +//typedef TarsHashMap PropHashMap; -// #if TARGET_PLATFORM_IOS || TARGET_PLATFORM_WINDOWS -// typedef TarsHashMap PropHashMap;//FileStorePolicy -// #else -// typedef TarsHashMap PropHashMap;//FileStorePolicy -// #endif +#if TARGET_PLATFORM_IOS +typedef TarsHashMap PropHashMap;//FileStorePolicy +#elif TARGET_PLATFORM_WINDOWS +typedef TarsHashMap PropHashMap;//FileStorePolicy +#else +typedef TarsHashMap PropHashMap;//FileStorePolicy +#endif #if TARGET_PLATFORM_LINUX diff --git a/PropertyServer/PropertyImp.cpp b/PropertyServer/PropertyImp.cpp index cc178686..ed2cfab6 100644 --- a/PropertyServer/PropertyImp.cpp +++ b/PropertyServer/PropertyImp.cpp @@ -152,10 +152,12 @@ int PropertyImp::handlePropMsg(const map &prop ////////////////////////////////////////////////////////////////////////////////////// float rate = (pHashMap->getMapHead()._iUsedChunk) * 1.0/pHashMap->allBlockChunkCount(); - if(rate >0.9) + if(rate >0.95) { - pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2); - TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl); +// pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2); + TLOGERROR("PropertyImp::handlePropMsg hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl); + return -1; +// TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl); } int iRet = pHashMap->add(tHead, body); @@ -207,7 +209,6 @@ void PropertyImp::dump2file() g_app.setSelectBufferIndex(iSelectBuffer); TLOGDEBUG("PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl; } } } diff --git a/PropertyServer/PropertyReapThread.cpp b/PropertyServer/PropertyReapThread.cpp index aec70218..cae52a71 100644 --- a/PropertyServer/PropertyReapThread.cpp +++ b/PropertyServer/PropertyReapThread.cpp @@ -96,12 +96,12 @@ void ReapSSDProcThread::run() TLOGDEBUG("ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index) << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl); - FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index) - << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl; +// FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index) +// << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl; if((iEnd - iBegin)/1000 > g_app.getInserInterv() * 40) { - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl; string sMsg("stat ip:"); sMsg += ServerConfig::LocalIp; sMsg += " ReapSSDProcThread::run write db:"; @@ -125,7 +125,7 @@ void ReapSSDProcThread::run() catch(exception& e) { TLOGERROR("ReapSSDProcThread::run exception:" << e.what() << endl); - FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl; +// FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl; } } } @@ -183,7 +183,7 @@ void PropertyReapThread::run() int iLastIndex = -1; TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl; while (!_terminate) { @@ -211,10 +211,10 @@ void PropertyReapThread::run() int64_t tEnd = TNOWMS; TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl; TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl; if(iTotalNum <= 0) { @@ -284,7 +284,7 @@ void PropertyReapThread::run() } TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl; } } @@ -384,7 +384,7 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector &vAl continue ; } - FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl; +// FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl; PropertyHashMap::lock_iterator it = pHashMap->beginSetTime(); while ( it != pHashMap->end() ) @@ -428,12 +428,12 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector &vAl iTotalNum = iCount; TLOGDEBUG("PropertyReapThread::getDataFromBuffer Buffer Index:" << iIndex << "|get total size:" << iCount << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl; } catch (exception& ex) { TLOGERROR("PropertyReapThread::getDataFromBuffer exception:" << ex.what() << endl); - FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl; +// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl; string sMsg("PropertyReapThread::getDataFromBuffer Buffer Index:"); sMsg += TC_Common::tostr(iIndex); diff --git a/PropertyServer/PropertyServer.cpp b/PropertyServer/PropertyServer.cpp index 9db75e72..d4c9cc19 100644 --- a/PropertyServer/PropertyServer.cpp +++ b/PropertyServer/PropertyServer.cpp @@ -50,7 +50,6 @@ void PropertyServer::initialize() string s(""); _selectBuffer = getSelectBufferFromFlag(s); - for(size_t i =0; i < vec.size(); ++i) { vec[i].first = 0; @@ -58,7 +57,6 @@ void PropertyServer::initialize() } _buffer[_selectBuffer] = vec; - for(size_t i =0; i < vec.size(); ++i) { vec[i].first = 0; @@ -67,12 +65,10 @@ void PropertyServer::initialize() _buffer[!_selectBuffer] = vec; TLOGDEBUG("PropertyServer::initialize iHandleNum:" << iHandleNum<< endl); - FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl; - +// FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl; TLOGDEBUG("PropertyServer::initialize iSelectBuffer:" << _selectBuffer<< endl); - FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl; - +// FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl; _randOrder = AppCache::getInstance()->get("RandOrder"); TLOGDEBUG("PropertyServer::initialize randorder:" << _randOrder << endl); @@ -203,15 +199,6 @@ void PropertyServer::initHashMap() float iFactor = TC_Common::strto(g_pconf->get("/tars/hashmap","2")); int iSize = TC_Common::toSize(g_pconf->get("/tars/hashmap"), 1024*1024*256); - - // _clonePath = ServerConfig::DataPath + "/" + g_pconf->get("/tars/hashmap","clone"); - - // if(!TC_File::makeDirRecursive(_clonePath)) - // { - // TLOGERROR("cannot create hashmap file " << _clonePath << endl); - // exit(0); - // } - TLOGDEBUG("PropertyServer::initHashMap init multi hashmap begin..." << endl); for(int i = 0; i < 2; ++i) @@ -245,11 +232,14 @@ void PropertyServer::initHashMap() _hashmap[i][k].initDataBlockSize(iMinBlock,iMaxBlock,iFactor); - if(TC_File::isFileExist(sHashMapFile)) - { - iSize = TC_File::getFileSize(sHashMapFile); - } - _hashmap[i][k].initStore( sHashMapFile.c_str(), iSize ); +#if TARGET_PLATFORM_IOS + _hashmap[i][k].create(new char[iSize], iSize); +#elif TARGET_PLATFORM_WINDOWS + _hashmap[i][k].initStore(sHashMapFile.c_str(), iSize); +#else + key_t key = tars::hash()(sHashMapFile); + _hashmap[i][k].initStore(key, iSize); +#endif TLOGINFO("\n" << _hashmap[i][k].desc() << endl); } diff --git a/StatServer/ReapSSDThread.cpp b/StatServer/ReapSSDThread.cpp index 59819a1e..c1a5a9b3 100644 --- a/StatServer/ReapSSDThread.cpp +++ b/StatServer/ReapSSDThread.cpp @@ -95,12 +95,12 @@ void ReapSSDProcThread::run() TLOGDEBUG("ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << StatDbManager::getInstance()->getIpAndPort(item._index) << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl); - FDLOG("CountStat") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << StatDbManager::getInstance()->getIpAndPort(item._index) - << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl; +// FDLOG("CountStat") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << StatDbManager::getInstance()->getIpAndPort(item._index) +// << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl; if((iEnd - iBegin)/1000 > (g_app.getInserInterv() - 2) * 60) { - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl; string sMsg("stat ip:"); sMsg += ServerConfig::LocalIp; sMsg += " ReapSSDProcThread::run write db:"; @@ -124,7 +124,7 @@ void ReapSSDProcThread::run() catch(exception& e) { TLOGERROR("ReapSSDProcThread::run exception:" << e.what() << endl); - FDLOG("CountStat") << "ReapSSDProcThread::run exception:" << e.what() << endl; +// FDLOG("CountStat") << "ReapSSDProcThread::run exception:" << e.what() << endl; } } } @@ -181,7 +181,7 @@ void ReapSSDThread::run() int iLastIndex = -1; - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ReapSSDThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ReapSSDThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl; while (!_terminate) { @@ -209,10 +209,10 @@ void ReapSSDThread::run() int64_t tEnd = TNOWMS; TLOGDEBUG("stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl; TLOGDEBUG("stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run insert begin _vAllStatMsg.size:" << vAllStatMsg.size() << "|record num:" << iTotalNum << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run insert begin _vAllStatMsg.size:" << vAllStatMsg.size() << "|record num:" << iTotalNum << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|ReapSSDThread::run insert begin _vAllStatMsg.size:" << vAllStatMsg.size() << "|record num:" << iTotalNum << endl; if(iTotalNum <= 0) { @@ -382,7 +382,7 @@ void ReapSSDThread::getDataFromBuffer(int iIndex, vector &vAllStatMsg, continue ; } - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData load hashmap k:" << k << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData load hashmap k:" << k << endl; HashMap::lock_iterator it = pHashMap->beginSetTime(); while ( it != pHashMap->end() ) @@ -426,12 +426,12 @@ void ReapSSDThread::getDataFromBuffer(int iIndex, vector &vAllStatMsg, iTotalNum = iCount; TLOGDEBUG("ReapSSDThread::getDataFromBuffer Buffer Index:" << iIndex << "|get total size:" << iCount << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData get total size:" << iCount << "|end..." << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData get total size:" << iCount << "|end..." << endl; } catch (exception& ex) { TLOGERROR("ReapSSDThread::getDataFromBuffer exception:" << ex.what() << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData exception:" << ex.what() << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|ReapSSDThread::getData exception:" << ex.what() << endl; string sMsg("ReapSSDThread::getDataFromBuffer Buffer Index:"); sMsg += TC_Common::tostr(iIndex); diff --git a/StatServer/StatDbManager.cpp b/StatServer/StatDbManager.cpp index 2e84785a..ff5ebe76 100644 --- a/StatServer/StatDbManager.cpp +++ b/StatServer/StatDbManager.cpp @@ -161,7 +161,7 @@ StatDbManager::StatDbManager() { vector &vDb = m_iter->second; - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ip:" << m_iter->first << "|DbNum:" << vDb.size() << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|ip:" << m_iter->first << "|DbNum:" << vDb.size() << endl; ++m_iter; } @@ -391,7 +391,7 @@ int StatDbManager::insert2Db(const StatMsg &statmsg, const string &sDate, const pMysql->execute(sSql); TLOGDEBUG("insert " << sTbName << " affected:" << iCount << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert " << sTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert " << sTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; iCount = 0; } } @@ -400,7 +400,7 @@ int StatDbManager::insert2Db(const StatMsg &statmsg, const string &sDate, const { pMysql->execute(sSql); TLOGDEBUG("insert " << sTbName << " affected:" << iCount << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert " << sTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert " << sTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl; } } catch (TC_Mysql_Exception& ex) @@ -612,8 +612,8 @@ int StatDbManager::insert2MultiDbs(int iIndex, const StatMsg &statmsg, const str int64_t iEnd = tars::TC_TimeProvider::getInstance()->getNowMs(); TLOGDEBUG("insert|" << iIndex << "|" << getIpAndPort(iIndex) << "|" << sDate << "|" << sFlag << "|" << statmsg.size() << "|" << (iEnd - iBegin) << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag - << "|records:" << statmsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag +// << "|records:" << statmsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl; } } catch(TC_Mysql_Exception& ex) diff --git a/StatServer/StatHashMap.h b/StatServer/StatHashMap.h index 40b5f8d4..27aaf046 100644 --- a/StatServer/StatHashMap.h +++ b/StatServer/StatHashMap.h @@ -30,11 +30,13 @@ using namespace tars; //typedef TarsHashMap HashMap;//FileStorePolicy - #if TARGET_PLATFORM_IOS || TARGET_PLATFORM_WINDOWS - typedef TarsHashMap HashMap;//FileStorePolicy - #else - typedef TarsHashMap HashMap;//FileStorePolicy - #endif +#if TARGET_PLATFORM_IOS +typedef TarsHashMap HashMap;//FileStorePolicy +#elif TARGET_PLATFORM_WINDOWS +typedef TarsHashMap HashMap;//FileStorePolicy +#else +typedef TarsHashMap HashMap;//FileStorePolicy +#endif #if TARGET_PLAFFORM_LINUX #include diff --git a/StatServer/StatImp.cpp b/StatServer/StatImp.cpp index 6d237c02..f69c3965 100644 --- a/StatServer/StatImp.cpp +++ b/StatServer/StatImp.cpp @@ -213,7 +213,7 @@ int StatImp::addHashMap(const StatMicMsgHead &head, const StatMicMsgBody &body ) if(rate >0.9) { TLOGERROR("StatImp::addHashMap hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl); - FDLOG("HashMap")<<"StatImp::addHashMap hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl; +// FDLOG("HashMap")<<"StatImp::addHashMap hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl; return -1; } @@ -260,7 +260,7 @@ void StatImp::dump2file() g_app.setSelectBufferIndex(iSelectBuffer); TLOGDEBUG("StatImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl); - FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|StatImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl; +// FDLOG("CountStat") << "stat ip:" << ServerConfig::LocalIp << "|StatImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl; } } } diff --git a/StatServer/StatServer.cpp b/StatServer/StatServer.cpp index 33b21da2..4b5d2bec 100644 --- a/StatServer/StatServer.cpp +++ b/StatServer/StatServer.cpp @@ -187,25 +187,8 @@ void StatServer::initHashMap() float iFactor = TC_Common::strto(g_pconf->get("/tars/hashmap","2")); int iSize = TC_Common::toSize(g_pconf->get("/tars/hashmap"), 1024*1024*256); -// _sClonePath = ServerConfig::DataPath + "/" + g_pconf->get("/tars/hashmap","clone"); -// -// if(!TC_File::makeDirRecursive(_sClonePath)) -// { -// TLOGERROR("cannot create hashmap file " << _sClonePath << endl); -// exit(0); -// } - TLOGDEBUG("StatServer::initHashMap init multi hashmap begin..." << endl); -// #if TARGET_PLATFORM_LINUX - // char a[26]="\0"; - // int iChar = 0; - // for(int n = 0; n < 26; n++) - // { - // a[n] = 'a' + n; - // } -// #endif - for(int i = 0; i < 2; ++i) { for(int k = 0; k < iHashMapNum; ++k) @@ -237,53 +220,13 @@ void StatServer::initHashMap() _hashmap[i][k].initDataBlockSize(iMinBlock,iMaxBlock,iFactor); -// if(TC_File::isFileExist(sHashMapFile)) -// { -// iSize = TC_File::getFileSize(sHashMapFile); -// } -// else -// { -// int fd = open(sHashMapFile.c_str(), O_CREAT|O_EXCL|O_RDWR, 0666); -// if(fd == -1) -// { -// if(errno != EEXIST) -// { -// throw TC_Exception("open1 file '" + sHashMapFile + "' error", errno); -// } -// else -// { -// fd = open(sHashMapFile.c_str(), O_CREAT|O_RDWR, 0666); -// if(fd == -1) -// { -// throw TC_Exception("open2 file '" + sHashMapFile + "' error", errno); -// } -// } -// } -// -// lseek(fd, iSize-1, SEEK_SET); -// write(fd,"\0",1); -// if(fd != -1) -// { -// close(fd); -// } -// } - - - //_hashmap[i][k].initStore( sHashMapFile.c_str(), iSize ); -#if TARGET_PLATFORM_IOS || TARGET_PLATFORM_WINDOWS +#if TARGET_PLATFORM_IOS _hashmap[i][k].create(new char[iSize], iSize); +#elif TARGET_PLATFORM_WINDOWS + _hashmap[i][k].initStore(sHashMapFile.c_str(), iSize); #else - // key_t key = ftok(sHashMapFile.c_str(), a[iChar%26]); - - // _hashmap[i][k].initStore( sHashMapFile.c_str(), iSize ); - key_t key = tars::hash()(sHashMapFile); - -// // iChar++; - -// TLOGDEBUG("init hash mem，shm key: 0x" << hex << key << dec << endl); - - _hashmap[i][k].initStore(key, iSize); + _hashmap[i][k].initStore(key, iSize); #endif _hashmap[i][k].setAutoErase(false); diff --git a/deploy/framework/sql/template/tars.tarsproperty b/deploy/framework/sql/template/tars.tarsproperty index 0c9bf46f..8882fbd1 100755 --- a/deploy/framework/sql/template/tars.tarsproperty +++ b/deploy/framework/sql/template/tars.tarsproperty @@ -7,7 +7,7 @@ insertInterval=5 maxBlock=200 minBlock=100 - size=256M + size=128M Interval=10 diff --git a/deploy/framework/sql/template/tars.tarsstat b/deploy/framework/sql/template/tars.tarsstat index e36faf54..073f7446 100755 --- a/deploy/framework/sql/template/tars.tarsstat +++ b/deploy/framework/sql/template/tars.tarsstat @@ -12,7 +12,6 @@ insertInterval=5 enableStatCount=0 size=256M - countsize=1M diff --git a/tarscpp b/tarscpp index 467166aa..fc4578e0 160000 --- a/tarscpp +++ b/tarscpp @@ -1 +1 @@ -Subproject commit 467166aa425c411cf4ff586e18300f526bb4ca1d +Subproject commit fc4578e0c0c1d702bfdd953ed8743befafb2eced