From 06a8cb2d9c318772701d0097f8eacb5189f8d2d4 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Mon, 29 Mar 2021 10:40:11 -0400 Subject: [PATCH] Remove log printous of individual DAS GET/POST as they significantly increase log content --- das/das.go | 1 - services/combined.go | 9 --------- services/helpers.go | 4 ---- services/reqmgr.go | 2 -- utils/fetch.go | 20 +++++++++----------- 5 files changed, 9 insertions(+), 27 deletions(-) diff --git a/das/das.go b/das/das.go index e39f514..fdd390d 100644 --- a/das/das.go +++ b/das/das.go @@ -488,7 +488,6 @@ func processURLs(dasquery dasql.DASQuery, urls map[string]string, maps []mongo.D for { select { case r := <-out: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) system := "" expire := 0 urn := "" diff --git a/services/combined.go b/services/combined.go index 8a3416b..b5205fe 100644 --- a/services/combined.go +++ b/services/combined.go @@ -45,7 +45,6 @@ func (LocalAPIs) Child4SiteReleaseDataset(dasquery dasql.DASQuery) []mongo.DASRe furl := fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := DBSUnmarshal(api, resp.Data) // collect dbs urls to fetch versions for given set of datasets api = "releaseversions" @@ -140,7 +139,6 @@ func (LocalAPIs) Site4Block(dasquery dasql.DASQuery) []mongo.DASRecord { furl := fmt.Sprintf("%s/%s?block=%s", PhedexUrl(), api, url.QueryEscape(block)) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := PhedexUnmarshal(api, resp.Data) for _, rec := range records { if rec["replica"] == nil { @@ -240,7 +238,6 @@ func rucioInfo(dasquery dasql.DASQuery, blockNames []string) (mongo.DASRecord, m for { select { case r := <-chout: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) records := RucioUnmarshal(dasquery, "full_record", r.Data) // get block name from r.URL blkName := getBlockNameFromUrl(r.Url) @@ -350,7 +347,6 @@ func rucioInfoMID(dasquery dasql.DASQuery, blockNames []string) (mongo.DASRecord for { select { case r := <-chout: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) records := RucioUnmarshal(dasquery, "full_record", r.Data) // get block name from r.URL blkName := getBlockNameFromUrl(r.Url) @@ -434,7 +430,6 @@ func (LocalAPIs) Site4DatasetPct(dasquery dasql.DASQuery) []mongo.DASRecord { furl := fmt.Sprintf("%s/%s?dataset=%s&validFileOnly=1", DBSUrl(inst), api, dataset) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := DBSUnmarshal(api, resp.Data) var totblocks, totfiles int64 if len(records) == 0 { @@ -448,7 +443,6 @@ func (LocalAPIs) Site4DatasetPct(dasquery dasql.DASQuery) []mongo.DASRecord { api = "blocks" furl = fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset) resp = utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records = DBSUnmarshal(api, resp.Data) var blocks []string for _, rec := range records { @@ -517,7 +511,6 @@ func (LocalAPIs) Site4Dataset_phedex(dasquery dasql.DASQuery) []mongo.DASRecord furl := fmt.Sprintf("%s/%s?dataset=%s&validFileOnly=1", DBSUrl(inst), api, dataset) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := DBSUnmarshal(api, resp.Data) var totblocks, totfiles int64 if len(records) == 0 { @@ -529,7 +522,6 @@ func (LocalAPIs) Site4Dataset_phedex(dasquery dasql.DASQuery) []mongo.DASRecord api = "blockReplicas" furl = fmt.Sprintf("%s/%s?dataset=%s", PhedexUrl(), api, dataset) resp = utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records = PhedexUnmarshal(api, resp.Data) siteInfo := make(mongo.DASRecord) var bComplete, nfiles, nblks, bfiles int64 @@ -686,7 +678,6 @@ func filterFilesInRucio(dasquery dasql.DASQuery, files []string, dataset, site s furl := fmt.Sprintf("%s/replicas/list", RucioUrl()) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, string(args)) // POST request - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := RucioUnmarshal(dasquery, "full_record", resp.Data) for _, r := range records { if v, ok := r["name"]; ok { diff --git a/services/helpers.go b/services/helpers.go index 4a04ec8..7bdc9ff 100644 --- a/services/helpers.go +++ b/services/helpers.go @@ -90,7 +90,6 @@ func findBlocks(dasquery dasql.DASQuery) []string { furl := fmt.Sprintf("%s/%s?dataset=%s", DBSUrl(inst), api, dataset) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := DBSUnmarshal(api, resp.Data) for _, rec := range records { v := rec["block_name"] @@ -118,7 +117,6 @@ func processUrls(dasquery dasql.DASQuery, system, api string, urls []string) []m for { select { case r := <-out: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) // process data var records []mongo.DASRecord if system == "dbs3" || system == "dbs" { @@ -293,7 +291,6 @@ func dataset4release(dasquery dasql.DASQuery) []string { } client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("pid=%s %s\n", dasquery.Qhash, resp.Details()) records := DBSUnmarshal(api, resp.Data) for _, rec := range records { if rec["name"] == nil { @@ -378,7 +375,6 @@ func (p *PhedexNodes) Nodes() []mongo.DASRecord { furl := fmt.Sprintf("%s/%s", PhedexUrl(), api) client := utils.HttpClient() resp := utils.FetchResponse(client, furl, "") // "" specify optional args - log.Printf("phedexNodes %s\n", resp.Details()) p.nodes = PhedexUnmarshal(api, resp.Data) p.tstamp = time.Now().Unix() return p.nodes diff --git a/services/reqmgr.go b/services/reqmgr.go index 2f2f283..683a45e 100644 --- a/services/reqmgr.go +++ b/services/reqmgr.go @@ -161,7 +161,6 @@ func findReqMgrIds(dasquery dasql.DASQuery, base, dataset string) ([]string, map for { select { case r := <-ch: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) var data mongo.DASRecord view := "" if strings.Contains(strings.ToLower(r.Url), "inputdataset") { @@ -256,7 +255,6 @@ func reqmgrConfigs(dasquery dasql.DASQuery) []mongo.DASRecord { for { select { case r := <-ch: - log.Printf("pid=%s %s\n", dasquery.Qhash, r.Details()) var data mongo.DASRecord err := json.Unmarshal(r.Data, &data) if err == nil { diff --git a/utils/fetch.go b/utils/fetch.go index 934eaaa..363c7f3 100644 --- a/utils/fetch.go +++ b/utils/fetch.go @@ -369,31 +369,29 @@ func FetchResponse(httpClient *http.Client, rurl, args string) ResponseType { if err != nil { response.Error = err } - if args == "" { - if WEBSERVER == 0 { - if VERBOSE > 0 { + if VERBOSE > 0 { + if args == "" { + if WEBSERVER == 0 { r, e := url.QueryUnescape(rurl) if e == nil { fmt.Printf("DAS GET %s %v\n", r, time.Now().Sub(startTime)) } else { fmt.Printf("DAS GET %s %v\n", rurl, time.Now().Sub(startTime)) } + } else { + log.Printf("DAS GET system=%s url=\"%s\" time=%v\n", system(rurl), rurl, time.Now().Sub(startTime)) } - // } else { - // log.Printf("DAS GET system=%s url=\"%s\" time=%v\n", system(rurl), rurl, time.Now().Sub(startTime)) - } - } else { - if WEBSERVER == 0 { - if VERBOSE > 0 { + } else { + if WEBSERVER == 0 { r, e := url.QueryUnescape(rurl) if e == nil { fmt.Printf("DAS POST %s args %v, %v\n", r, args, time.Now().Sub(startTime)) } else { fmt.Printf("DAS POST %s args %v, %v\n", rurl, args, time.Now().Sub(startTime)) } + } else { + log.Printf("DAS POST system=%s url=\"%s\" args=\"%v\" time=%v\n", system(rurl), rurl, args, time.Now().Sub(startTime)) } - } else { - // log.Printf("DAS POST system=%s url=\"%s\" args=\"%v\" time=%v\n", system(rurl), rurl, args, time.Now().Sub(startTime)) } } return response