fix tarstat, tarsproperty, mac: mem, windows: mmap, linux: shm

TarsCloud · Mar 31, 2020 · f632739 · f632739
1 parent 5f0e831
commit f632739
Show file tree

Hide file tree

Showing 14 changed files with 70 additions and 133 deletions.
diff --git a/.github/workflows/ccpp-win.yml b/.github/workflows/ccpp-win.yml
@@ -22,5 +22,5 @@ jobs:
       - name: configure
         run: cd build; cmake --version; cmake .. -A X64
       - name: make
-        run: cd build; cmake --build . --config release --target run-all
+        run: cd build; cmake --build . --config release
 
diff --git a/PropertyServer/PropertyDbManager.cpp b/PropertyServer/PropertyDbManager.cpp
@@ -207,7 +207,7 @@ PropertyDbManager::PropertyDbManager()
     {
         vector<size_t> &vDb = m_iter1->second;
 
-        FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl;
+//        FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl;
 
         ++m_iter1;
     }
@@ -380,7 +380,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat
 
                 TLOGDEBUG("insert " << strTbName << " affected:" << iCount  << endl);
 
-                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
+//                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
 
                 iCount = 0;
             }
@@ -390,7 +390,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat
         {
             pMysql->execute(osSql.str());
             TLOGDEBUG("insert " << strTbName << " affected:" << iCount << endl);
-            FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
+//            FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
         }
     }
     catch (TC_Mysql_Exception& ex)
@@ -597,8 +597,8 @@ int PropertyDbManager::insert2MultiDbs(int iIndex, const PropertyMsg &propertyms
             int64_t iEnd = tars::TC_TimeProvider::getInstance()->getNowMs();
 
             TLOGDEBUG("insert|" << iIndex << "|" << getIpAndPort(iIndex) << "|" << sDate << "|" << sFlag << "|" << propertymsg.size() << "|" << (iEnd - iBegin) << endl);
-            FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp  << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag 
-                << "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl;
+//            FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp  << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag
+//                << "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl;
         }
     }
     catch(TC_Mysql_Exception& ex)

diff --git a/PropertyServer/PropertyHashMap.h b/PropertyServer/PropertyHashMap.h
@@ -27,13 +27,15 @@ using namespace tars;
 typedef StatPropMsgBody PropBody;
 typedef StatPropMsgHead PropHead;
 
-typedef TarsHashMap<PropHead,PropBody, ThreadLockPolicy, FileStorePolicy> PropHashMap;
+//typedef TarsHashMap<PropHead,PropBody, ThreadLockPolicy, FileStorePolicy> PropHashMap;
 
-// #if TARGET_PLATFORM_IOS || TARGET_PLATFORM_WINDOWS
-// typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,MemStorePolicy> PropHashMap;//FileStorePolicy
-// #else
-// typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,ShmStorePolicy> PropHashMap;//FileStorePolicy
-// #endif
+#if TARGET_PLATFORM_IOS
+typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,MemStorePolicy> PropHashMap;//FileStorePolicy
+#elif TARGET_PLATFORM_WINDOWS
+typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,FileStorePolicy> PropHashMap;//FileStorePolicy
+#else
+typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,ShmStorePolicy> PropHashMap;//FileStorePolicy
+#endif
 
 
 #if TARGET_PLATFORM_LINUX

diff --git a/PropertyServer/PropertyImp.cpp b/PropertyServer/PropertyImp.cpp
@@ -152,10 +152,12 @@ int PropertyImp::handlePropMsg(const map<StatPropMsgHead, StatPropMsgBody> &prop
         //////////////////////////////////////////////////////////////////////////////////////
         float rate =  (pHashMap->getMapHead()._iUsedChunk) * 1.0/pHashMap->allBlockChunkCount();
 
-        if(rate >0.9)
+        if(rate >0.95)
         {
-            pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2);
-            TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl);
+//            pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2);
+	        TLOGERROR("PropertyImp::handlePropMsg hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl);
+	        return -1;
+//            TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl);
         }
 
         int iRet = pHashMap->add(tHead, body);
@@ -207,7 +209,6 @@ void PropertyImp::dump2file()
             g_app.setSelectBufferIndex(iSelectBuffer);
 
             TLOGDEBUG("PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl);
-            FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl;
         }
     }
 }

diff --git a/PropertyServer/PropertyReapThread.cpp b/PropertyServer/PropertyReapThread.cpp
@@ -96,12 +96,12 @@ void ReapSSDProcThread::run()
                     TLOGDEBUG("ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp  << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index) 
                         << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl);
 
-                    FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp  << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index) 
-                        << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl;
+//                    FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp  << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index)
+//                        << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl;
 
                     if((iEnd - iBegin)/1000 > g_app.getInserInterv() * 40)
                     {
-                        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl;
+//                        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl;
                         string sMsg("stat ip:");
                         sMsg += ServerConfig::LocalIp;
                         sMsg += " ReapSSDProcThread::run write db:";
@@ -125,7 +125,7 @@ void ReapSSDProcThread::run()
         catch(exception& e)
         {
             TLOGERROR("ReapSSDProcThread::run exception:" << e.what() << endl);
-            FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl;
+//            FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl;
         }
     }
 }
@@ -183,7 +183,7 @@ void PropertyReapThread::run()
     int iLastIndex = -1;
 
     TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl);
-    FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl;
+//    FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl;
 
     while (!_terminate)
     {
@@ -211,10 +211,10 @@ void PropertyReapThread::run()
                 int64_t tEnd = TNOWMS;
 
                 TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl);
-                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl;
+//                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl;
 
                 TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl);
-                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl;
+//                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl;
 
                 if(iTotalNum <= 0)
                 {
@@ -284,7 +284,7 @@ void PropertyReapThread::run()
                 }
 
                 TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl);
-                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl;
+//                FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl;
             }
 
         }
@@ -384,7 +384,7 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector<PropertyMsg*> &vAl
                 continue ;
             }
 
-            FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl;
+//            FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl;
 
             PropertyHashMap::lock_iterator it = pHashMap->beginSetTime();
             while ( it != pHashMap->end() )
@@ -428,12 +428,12 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector<PropertyMsg*> &vAl
         iTotalNum = iCount;
 
         TLOGDEBUG("PropertyReapThread::getDataFromBuffer Buffer Index:" << iIndex << "|get total size:" << iCount << endl);
-        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl;
+//        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl;
     }
     catch (exception& ex)
     {
         TLOGERROR("PropertyReapThread::getDataFromBuffer exception:" << ex.what() << endl);
-        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl;
+//        FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl;
 
         string sMsg("PropertyReapThread::getDataFromBuffer Buffer Index:");
         sMsg += TC_Common::tostr(iIndex);

diff --git a/PropertyServer/PropertyServer.cpp b/PropertyServer/PropertyServer.cpp
@@ -50,15 +50,13 @@ void PropertyServer::initialize()
         string s("");
         _selectBuffer = getSelectBufferFromFlag(s);
 
-
         for(size_t i =0; i < vec.size(); ++i)
         {
             vec[i].first = 0;
             vec[i].second = 0;
         }
         _buffer[_selectBuffer] = vec;
 
-
         for(size_t i =0; i < vec.size(); ++i)
         {
             vec[i].first = 0;
@@ -67,12 +65,10 @@ void PropertyServer::initialize()
         _buffer[!_selectBuffer] = vec;
 
         TLOGDEBUG("PropertyServer::initialize iHandleNum:" << iHandleNum<< endl);
-        FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl;
-
+//        FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl;
 
         TLOGDEBUG("PropertyServer::initialize iSelectBuffer:" << _selectBuffer<< endl);
-        FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl;
-
+//        FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl;
 
         _randOrder = AppCache::getInstance()->get("RandOrder");
         TLOGDEBUG("PropertyServer::initialize randorder:" << _randOrder << endl);
@@ -203,15 +199,6 @@ void PropertyServer::initHashMap()
     float iFactor       = TC_Common::strto<float>(g_pconf->get("/tars/hashmap<factor>","2"));
     int iSize           = TC_Common::toSize(g_pconf->get("/tars/hashmap<size>"), 1024*1024*256);
 
-
-    // _clonePath         = ServerConfig::DataPath + "/" + g_pconf->get("/tars/hashmap<clonePatch>","clone");
-
-    // if(!TC_File::makeDirRecursive(_clonePath))
-    // {
-    //     TLOGERROR("cannot create hashmap file " << _clonePath << endl);
-    //     exit(0);
-    // }
-
     TLOGDEBUG("PropertyServer::initHashMap init multi hashmap begin..." << endl);
 
     for(int i = 0; i < 2; ++i)
@@ -245,11 +232,14 @@ void PropertyServer::initHashMap()
 
                 _hashmap[i][k].initDataBlockSize(iMinBlock,iMaxBlock,iFactor);
 
-                if(TC_File::isFileExist(sHashMapFile))
-                {
-                    iSize = TC_File::getFileSize(sHashMapFile);
-                }
-                _hashmap[i][k].initStore( sHashMapFile.c_str(), iSize );
+#if TARGET_PLATFORM_IOS
+	            _hashmap[i][k].create(new char[iSize], iSize);
+#elif TARGET_PLATFORM_WINDOWS
+	            _hashmap[i][k].initStore(sHashMapFile.c_str(), iSize);
+#else
+               key_t key = tars::hash<string>()(sHashMapFile);
+               _hashmap[i][k].initStore(key, iSize);
+#endif
 
                 TLOGINFO("\n" <<  _hashmap[i][k].desc() << endl);
             }