Remove log printous of individual DAS GET/POST as they significantly …

…increase log content
dmwm · Mar 29, 2021 · 06a8cb2 · 06a8cb2
1 parent 20818d1
commit 06a8cb2
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 27 deletions.
diff --git a/das/das.go b/das/das.go
@@ -488,7 +488,6 @@ func processURLs(dasquery dasql.DASQuery, urls map[string]string, maps []mongo.D
 	for {
 		select {
 		case r := <-out:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			system := ""
 			expire := 0
 			urn := ""

diff --git a/services/combined.go b/services/combined.go
@@ -45,7 +45,6 @@ func (LocalAPIs) Child4SiteReleaseDataset(dasquery dasql.DASQuery) []mongo.DASRe
 	furl := fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset)
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := DBSUnmarshal(api, resp.Data)
 	// collect dbs urls to fetch versions for given set of datasets
 	api = "releaseversions"
@@ -140,7 +139,6 @@ func (LocalAPIs) Site4Block(dasquery dasql.DASQuery) []mongo.DASRecord {
 	furl := fmt.Sprintf("%s/%s?block=%s", PhedexUrl(), api, url.QueryEscape(block))
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := PhedexUnmarshal(api, resp.Data)
 	for _, rec := range records {
 		if rec["replica"] == nil {
@@ -240,7 +238,6 @@ func rucioInfo(dasquery dasql.DASQuery, blockNames []string) (mongo.DASRecord, m
 	for {
 		select {
 		case r := <-chout:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			records := RucioUnmarshal(dasquery, "full_record", r.Data)
 			// get block name from r.URL
 			blkName := getBlockNameFromUrl(r.Url)
@@ -350,7 +347,6 @@ func rucioInfoMID(dasquery dasql.DASQuery, blockNames []string) (mongo.DASRecord
 	for {
 		select {
 		case r := <-chout:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			records := RucioUnmarshal(dasquery, "full_record", r.Data)
 			// get block name from r.URL
 			blkName := getBlockNameFromUrl(r.Url)
@@ -434,7 +430,6 @@ func (LocalAPIs) Site4DatasetPct(dasquery dasql.DASQuery) []mongo.DASRecord {
 	furl := fmt.Sprintf("%s/%s?dataset=%s&validFileOnly=1", DBSUrl(inst), api, dataset)
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := DBSUnmarshal(api, resp.Data)
 	var totblocks, totfiles int64
 	if len(records) == 0 {
@@ -448,7 +443,6 @@ func (LocalAPIs) Site4DatasetPct(dasquery dasql.DASQuery) []mongo.DASRecord {
 	api = "blocks"
 	furl = fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset)
 	resp = utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records = DBSUnmarshal(api, resp.Data)
 	var blocks []string
 	for _, rec := range records {
@@ -517,7 +511,6 @@ func (LocalAPIs) Site4Dataset_phedex(dasquery dasql.DASQuery) []mongo.DASRecord
 	furl := fmt.Sprintf("%s/%s?dataset=%s&validFileOnly=1", DBSUrl(inst), api, dataset)
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := DBSUnmarshal(api, resp.Data)
 	var totblocks, totfiles int64
 	if len(records) == 0 {
@@ -529,7 +522,6 @@ func (LocalAPIs) Site4Dataset_phedex(dasquery dasql.DASQuery) []mongo.DASRecord
 	api = "blockReplicas"
 	furl = fmt.Sprintf("%s/%s?dataset=%s", PhedexUrl(), api, dataset)
 	resp = utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records = PhedexUnmarshal(api, resp.Data)
 	siteInfo := make(mongo.DASRecord)
 	var bComplete, nfiles, nblks, bfiles int64
@@ -686,7 +678,6 @@ func filterFilesInRucio(dasquery dasql.DASQuery, files []string, dataset, site s
 	furl := fmt.Sprintf("%s/replicas/list", RucioUrl())
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, string(args)) // POST request
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := RucioUnmarshal(dasquery, "full_record", resp.Data)
 	for _, r := range records {
 		if v, ok := r["name"]; ok {

diff --git a/services/helpers.go b/services/helpers.go
@@ -90,7 +90,6 @@ func findBlocks(dasquery dasql.DASQuery) []string {
 	furl := fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset)
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := DBSUnmarshal(api, resp.Data)
 	for _, rec := range records {
 		v := rec["block_name"]
@@ -118,7 +117,6 @@ func processUrls(dasquery dasql.DASQuery, system, api string, urls []string) []m
 	for {
 		select {
 		case r := <-out:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			// process data
 			var records []mongo.DASRecord
 			if system == "dbs3" || system == "dbs" {
@@ -293,7 +291,6 @@ func dataset4release(dasquery dasql.DASQuery) []string {
 	}
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details())
 	records := DBSUnmarshal(api, resp.Data)
 	for _, rec := range records {
 		if rec["name"] == nil {
@@ -378,7 +375,6 @@ func (p *PhedexNodes) Nodes() []mongo.DASRecord {
 	furl := fmt.Sprintf("%s/%s", PhedexUrl(), api)
 	client := utils.HttpClient()
 	resp := utils.FetchResponse(client, furl, "") // "" specify optional args
-	log.Printf("phedexNodes %s\n", resp.Details())
 	p.nodes = PhedexUnmarshal(api, resp.Data)
 	p.tstamp = time.Now().Unix()
 	return p.nodes

diff --git a/services/reqmgr.go b/services/reqmgr.go
@@ -161,7 +161,6 @@ func findReqMgrIds(dasquery dasql.DASQuery, base, dataset string) ([]string, map
 	for {
 		select {
 		case r := <-ch:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			var data mongo.DASRecord
 			view := ""
 			if strings.Contains(strings.ToLower(r.Url), "inputdataset") {
@@ -256,7 +255,6 @@ func reqmgrConfigs(dasquery dasql.DASQuery) []mongo.DASRecord {
 	for {
 		select {
 		case r := <-ch:
-			log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details())
 			var data mongo.DASRecord
 			err := json.Unmarshal(r.Data, &data)
 			if err == nil {

diff --git a/utils/fetch.go b/utils/fetch.go
@@ -369,31 +369,29 @@ func FetchResponse(httpClient *http.Client, rurl, args string) ResponseType {
 	if err != nil {
 		response.Error = err
 	}
-	if args == "" {
-		if WEBSERVER == 0 {
-			if VERBOSE > 0 {
+	if VERBOSE > 0 {
+		if args == "" {
+			if WEBSERVER == 0 {
 				r, e := url.QueryUnescape(rurl)
 				if e == nil {
 					fmt.Printf("DAS GET %s %v\n", r, time.Now().Sub(startTime))
 				} else {
 					fmt.Printf("DAS GET %s %v\n", rurl, time.Now().Sub(startTime))
 				}
+			} else {
+				log.Printf("DAS GET system=%s url=\"%s\" time=%v\n", system(rurl), rurl, time.Now().Sub(startTime))
 			}
-			//         } else {
-			//             log.Printf("DAS GET system=%s url=\"%s\" time=%v\n", system(rurl), rurl, time.Now().Sub(startTime))
-		}
-	} else {
-		if WEBSERVER == 0 {
-			if VERBOSE > 0 {
+		} else {
+			if WEBSERVER == 0 {
 				r, e := url.QueryUnescape(rurl)
 				if e == nil {
 					fmt.Printf("DAS POST %s args %v, %v\n", r, args, time.Now().Sub(startTime))
 				} else {
 					fmt.Printf("DAS POST %s args %v, %v\n", rurl, args, time.Now().Sub(startTime))
 				}
+			} else {
+				log.Printf("DAS POST system=%s url=\"%s\" args=\"%v\" time=%v\n", system(rurl), rurl, args, time.Now().Sub(startTime))
 			}
-		} else {
-			//             log.Printf("DAS POST system=%s url=\"%s\" args=\"%v\" time=%v\n", system(rurl), rurl, args, time.Now().Sub(startTime))
 		}
 	}
 	return response