diff --git a/database/database.go b/database/database.go index 724a7b9..088697e 100644 --- a/database/database.go +++ b/database/database.go @@ -47,11 +47,11 @@ func InitDb() { // createTable in database if not abvailable func createTable() { // Create platform, pipeline and job table - pipeline := []string{"packet_pipeline_v15", "packet_pipeline_v14", "packet_pipeline_v13", "konvoy_pipeline"} - pipelineJobs := []string{"packet_jobs_v15", "packet_jobs_v14", "packet_jobs_v13", "konvoy_jobs"} + pipeline := []string{"packet_pipeline_k8s_ultimate", "packet_pipeline_k8s_penultimate", "packet_pipeline_k8s_antepenultimate", "konvoy_pipeline", "release_pipeline_data"} + pipelineJobs := []string{"packet_jobs_k8s_ultimate", "packet_jobs_k8s_penultimate", "packet_jobs_k8s_antepenultimate", "konvoy_jobs", "release_jobs_data"} // Create pipeline table in database for i := range pipeline { - query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(build_pipelineid INT PRIMARY KEY, id INT, sha VARCHAR, ref VARCHAR, status VARCHAR, web_url VARCHAR, kibana_url VARCHAR);", pipeline[i]) + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(project VARCHAR, id INT PRIMARY KEY, sha VARCHAR, ref VARCHAR, status VARCHAR, web_url VARCHAR, openshift_pid VARCHAR, kibana_url VARCHAR, release_tag VARCHAR);", pipeline[i]) value, err := Db.Query(query) if err != nil { glog.Error(err) @@ -60,42 +60,13 @@ func createTable() { } // Create pipeline jobs table in database for i := range pipelineJobs { - query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(pipelineid INT, id INT PRIMARY KEY,status VARCHAR, stage VARCHAR, name VARCHAR, ref VARCHAR, created_at VARCHAR, started_at VARCHAR, finished_at VARCHAR, job_log_url VARCHAR);", pipelineJobs[i]) + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(pipelineid INT, id INT PRIMARY KEY,status VARCHAR, stage VARCHAR, name VARCHAR, ref VARCHAR, created_at VARCHAR, started_at VARCHAR, finished_at VARCHAR, message VARCHAR, author_name VARCHAR);", pipelineJobs[i]) value, err := Db.Query(query) if err != nil { glog.Error(err) } defer value.Close() } - // create build pipelines table for build related r/w operation - query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS build_pipeline(project VARCHAR, id INT PRIMARY KEY, sha VARCHAR, ref VARCHAR, status VARCHAR, web_url VARCHAR, packet_v15_pid VARCHAR, packet_v14_pid VARCHAR, packet_v13_pid VARCHAR, konvoy_pid VARCHAR);") - value, err := Db.Query(query) - if err != nil { - glog.Error(err) - } - defer value.Close() - - // create build pipeline jobs table in database - query = fmt.Sprintf("CREATE TABLE IF NOT EXISTS build_jobs(pipelineid INT, id INT PRIMARY KEY,status VARCHAR, stage VARCHAR, name VARCHAR, ref VARCHAR, created_at VARCHAR, started_at VARCHAR, finished_at VARCHAR, message VARCHAR, author_name VARCHAR);") - value, err = Db.Query(query) - if err != nil { - glog.Error(err) - } - defer value.Close() // create build pipelines table for build related r/w operation - query = fmt.Sprintf("CREATE TABLE IF NOT EXISTS release_pipeline_data(project VARCHAR, id INT PRIMARY KEY, sha VARCHAR, ref VARCHAR, status VARCHAR, web_url VARCHAR, openshift_pid VARCHAR, kibana_url VARCHAR, release_tag VARCHAR);") - value, err = Db.Query(query) - if err != nil { - glog.Error(err) - } - defer value.Close() - // create build pipeline jobs table in database - query = fmt.Sprintf("CREATE TABLE IF NOT EXISTS release_jobs_data(pipelineid INT, id INT PRIMARY KEY,status VARCHAR, stage VARCHAR, name VARCHAR, ref VARCHAR, created_at VARCHAR, started_at VARCHAR, finished_at VARCHAR, message VARCHAR, author_name VARCHAR);") - value, err = Db.Query(query) - if err != nil { - glog.Error(err) - } - defer value.Close() - } // dbConfig get config from environment variable diff --git a/handler/build.go b/handler/build.go deleted file mode 100644 index 217f23f..0000000 --- a/handler/build.go +++ /dev/null @@ -1,598 +0,0 @@ -package handler - -import ( - "encoding/json" - "io/ioutil" - "net/http" - "os/exec" - "strconv" - "strings" - - "github.com/golang/glog" - "github.com/mayadata-io/ci-e2e-status/database" -) - -// Buildhandler return eks pipeline data to /build path -func Buildhandler(w http.ResponseWriter, r *http.Request) { - // Allow cross origin request - (w).Header().Set("Access-Control-Allow-Origin", "*") - datas := Builddashboard{} - err := queryBuildData(&datas) - if err != nil { - http.Error(w, err.Error(), 500) - glog.Error(err) - return - } - out, err := json.Marshal(datas) - if err != nil { - http.Error(w, err.Error(), 500) - glog.Error(err) - return - } - w.Write(out) -} - -// BuildData from gitlab api and store to database -func BuildData(token string) { - jivaPipelineData, err := pipelineData("jiva", token) - if err != nil { - glog.Error(err) - return - } - project = "jiva" - for i := range jivaPipelineData { - jivaJobsData, err := pipelineJobsData(jivaPipelineData[i].ID, token, "jiva") - if err != nil { - glog.Error(err) - return - } - // Getting webURL link for getting triggredID - baselineJobsWebURL := getBaselineJobWebURL(jivaJobsData) - // Get GKE, Triggred pipeline ID for jiva build - packetV15PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-antepenultimate") - if err != nil { - glog.Error(err) - } - // Get EKS, Triggred pipeline ID for jiva build - packetV14PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-penultimate") - if err != nil { - glog.Error(err) - } - // Get AKS, Triggred pipeline ID for jiva build - packetV13PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-ultimate") - if err != nil { - glog.Error(err) - } - konvoyPID, err := getTriggerPipelineid(baselineJobsWebURL, "konvoy") - if err != nil { - glog.Error(err) - } - // Add jiva pipelines data to Database - sqlStatement := ` - INSERT INTO build_pipeline (project, id, sha, ref, status, web_url, packet_v15_pid, packet_v14_pid, packet_v13_pid, konvoy_pid) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE - SET status = $5, packet_v15_pid = $7, packet_v14_pid = $8, packet_v13_pid = $9, konvoy_pid = $10 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - project, - jivaPipelineData[i].ID, - jivaPipelineData[i].Sha, - jivaPipelineData[i].Ref, - jivaPipelineData[i].Status, - jivaPipelineData[i].WebURL, - packetV15PID, - packetV14PID, - packetV13PID, - konvoyPID, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for jiva Pipeline:", id) - - // Add jiva jobs data to Database - for j := range jivaJobsData { - sqlStatement := ` - INSERT INTO build_jobs (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, message, author_name) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - ON CONFLICT (id) DO UPDATE - SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - jivaPipelineData[i].ID, - jivaJobsData[j].ID, - jivaJobsData[j].Status, - jivaJobsData[j].Stage, - jivaJobsData[j].Name, - jivaJobsData[j].Ref, - jivaJobsData[j].CreatedAt, - jivaJobsData[j].StartedAt, - jivaJobsData[j].FinishedAt, - jivaJobsData[j].Commit.Message, - jivaJobsData[j].Commit.AuthorName, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for jiva pipeline Jobs: ", id) - } - } - - project = "maya" - mayaPipelineData, err := pipelineData("maya", token) - if err != nil { - glog.Error(err) - return - } - for i := range mayaPipelineData { - mayaJobsData, err := pipelineJobsData(mayaPipelineData[i].ID, token, "maya") - if err != nil { - glog.Error(err) - return - } - // Getting webURL link for getting triggredID - baselineJobsWebURL := getBaselineJobWebURL(mayaJobsData) - // Get GKE, Triggred pipeline ID for maya build - packetV15PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-antepenultimate") - if err != nil { - glog.Error(err) - } - // Get EKS, Triggred pipeline ID for maya build - packetV14PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-penultimate") - if err != nil { - glog.Error(err) - } - // Get AKS, Triggred pipeline ID for maya build - packetV13PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-ultimate") - if err != nil { - glog.Error(err) - } - konvoyPID, err := getTriggerPipelineid(baselineJobsWebURL, "konvoy-onprem") - if err != nil { - glog.Error(err) - } - // Add maya pipelines data to Database - sqlStatement := ` - INSERT INTO build_pipeline (project, id, sha, ref, status, web_url, packet_v15_pid, packet_v14_pid, packet_v13_pid, konvoy_pid) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE - SET status = $5, packet_v15_pid = $7, packet_v14_pid = $8, packet_v13_pid = $9, konvoy_pid = $10 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - project, - mayaPipelineData[i].ID, - mayaPipelineData[i].Sha, - mayaPipelineData[i].Ref, - mayaPipelineData[i].Status, - mayaPipelineData[i].WebURL, - packetV15PID, - packetV14PID, - packetV13PID, - konvoyPID, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for maya Pipeline:", id) - - // Add maya jobs data to Database - for j := range mayaJobsData { - sqlStatement := ` - INSERT INTO build_jobs (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, message, author_name) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - ON CONFLICT (id) DO UPDATE - SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - mayaPipelineData[i].ID, - mayaJobsData[j].ID, - mayaJobsData[j].Status, - mayaJobsData[j].Stage, - mayaJobsData[j].Name, - mayaJobsData[j].Ref, - mayaJobsData[j].CreatedAt, - mayaJobsData[j].StartedAt, - mayaJobsData[j].FinishedAt, - mayaJobsData[j].Commit.Message, - mayaJobsData[j].Commit.AuthorName, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for maya pipeline Jobs: ", id) - } - } - - project = "zfs" - zfsPipelineData, err := pipelineData("zfs", token) - if err != nil { - glog.Error(err) - return - } - for i := range zfsPipelineData { - zfsJobsData, err := pipelineJobsData(zfsPipelineData[i].ID, token, "zfs") - if err != nil { - glog.Error(err) - return - } - // Getting webURL link for getting triggredID - baselineJobsWebURL := getBaselineJobWebURL(zfsJobsData) - // Get GKE, Triggred pipeline ID for zfs build - packetV15PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-antepenultimate") - if err != nil { - glog.Error(err) - } - // Get EKS, Triggred pipeline ID for zfs build - packetV14PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-penultimate") - if err != nil { - glog.Error(err) - } - // Get AKS, Triggred pipeline ID for zfs build - packetV13PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-ultimate") - if err != nil { - glog.Error(err) - } - konvoyPID, err := getTriggerPipelineid(baselineJobsWebURL, "konvoy") - if err != nil { - glog.Error(err) - } - // Add zfs pipelines data to Database - sqlStatement := ` - INSERT INTO build_pipeline (project, id, sha, ref, status, web_url, packet_v15_pid, packet_v14_pid, packet_v13_pid, konvoy_pid) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE - SET status = $5, packet_v15_pid = $7, packet_v14_pid = $8, packet_v13_pid = $9, konvoy_pid = $10 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - project, - zfsPipelineData[i].ID, - zfsPipelineData[i].Sha, - zfsPipelineData[i].Ref, - zfsPipelineData[i].Status, - zfsPipelineData[i].WebURL, - packetV15PID, - packetV14PID, - packetV13PID, - konvoyPID, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for zfs Pipeline:", id) - - // Add zfs jobs data to Database - for j := range zfsJobsData { - sqlStatement := ` - INSERT INTO build_jobs (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, message, author_name) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - ON CONFLICT (id) DO UPDATE - SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - zfsPipelineData[i].ID, - zfsJobsData[j].ID, - zfsJobsData[j].Status, - zfsJobsData[j].Stage, - zfsJobsData[j].Name, - zfsJobsData[j].Ref, - zfsJobsData[j].CreatedAt, - zfsJobsData[j].StartedAt, - zfsJobsData[j].FinishedAt, - zfsJobsData[j].Commit.Message, - zfsJobsData[j].Commit.AuthorName, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for zfs pipeline Jobs: ", id) - } - } - - project = "istgt" - istgtPipelineData, err := pipelineData("istgt", token) - if err != nil { - glog.Error(err) - return - } - for i := range istgtPipelineData { - istgtJobsData, err := pipelineJobsData(istgtPipelineData[i].ID, token, "istgt") - if err != nil { - glog.Error(err) - return - } - // Getting webURL link for getting triggredID - baselineJobsWebURL := getBaselineJobWebURL(istgtJobsData) - // Get GKE, Triggred pipeline ID for istgt build - packetV15PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-antepenultimate") - if err != nil { - glog.Error(err) - } - // Get EKS, Triggred pipeline ID for istgt build - packetV14PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-penultimate") - if err != nil { - glog.Error(err) - } - // Get AKS, Triggred pipeline ID for istgt build - packetV13PID, err := getTriggerPipelineid(baselineJobsWebURL, "k8s-ultimate") - if err != nil { - glog.Error(err) - } - konvoyPID, err := getTriggerPipelineid(baselineJobsWebURL, "konvoy") - if err != nil { - glog.Error(err) - } - // Add istgt pipelines data to Database - sqlStatement := ` - INSERT INTO build_pipeline (project, id, sha, ref, status, web_url, packet_v15_pid, packet_v14_pid, packet_v13_pid, konvoy_pid) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE - SET status = $5, packet_v15_pid = $7, packet_v14_pid = $8, packet_v13_pid = $9, konvoy_pid = $10 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - project, - istgtPipelineData[i].ID, - istgtPipelineData[i].Sha, - istgtPipelineData[i].Ref, - istgtPipelineData[i].Status, - istgtPipelineData[i].WebURL, - packetV15PID, - packetV14PID, - packetV13PID, - konvoyPID, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for istgt Pipeline:", id) - - // Add istgt jobs data to Database - for j := range istgtJobsData { - sqlStatement := ` - INSERT INTO build_jobs (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, message, author_name) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - ON CONFLICT (id) DO UPDATE - SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 - RETURNING id` - id := 0 - err = database.Db.QueryRow(sqlStatement, - istgtPipelineData[i].ID, - istgtJobsData[j].ID, - istgtJobsData[j].Status, - istgtJobsData[j].Stage, - istgtJobsData[j].Name, - istgtJobsData[j].Ref, - istgtJobsData[j].CreatedAt, - istgtJobsData[j].StartedAt, - istgtJobsData[j].FinishedAt, - istgtJobsData[j].Commit.Message, - istgtJobsData[j].Commit.AuthorName, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for istgt pipeline Jobs:", id) - } - } - err = modifyBuildData() - if err != nil { - glog.Error(err) - } -} - -func modifyBuildData() error { - query, err := database.Db.Query(`DELETE FROM build_pipeline WHERE id < (SELECT id FROM build_pipeline ORDER BY id DESC LIMIT 1 OFFSET 19)`) - if err != nil { - return err - } - defer query.Close() - return nil -} - -// queryBuildData fetches the builddashboard data from the db -func queryBuildData(datas *Builddashboard) error { - pipelinerows, err := database.Db.Query(`SELECT * FROM build_pipeline ORDER BY id DESC`) - if err != nil { - return err - } - defer pipelinerows.Close() - for pipelinerows.Next() { - pipelinedata := BuildpipelineSummary{} - err = pipelinerows.Scan( - &pipelinedata.Project, - &pipelinedata.ID, - &pipelinedata.Sha, - &pipelinedata.Ref, - &pipelinedata.Status, - &pipelinedata.WebURL, - &pipelinedata.PacketV15PID, - &pipelinedata.PacketV14PID, - &pipelinedata.PacketV13PID, - &pipelinedata.KonvoyPID, - ) - if err != nil { - return err - } - - jobsquery := `SELECT * FROM build_jobs WHERE pipelineid = $1 ORDER BY id` - jobsrows, err := database.Db.Query(jobsquery, pipelinedata.ID) - if err != nil { - return err - } - defer jobsrows.Close() - jobsdataarray := []BuildJobssummary{} - for jobsrows.Next() { - jobsdata := BuildJobssummary{} - err = jobsrows.Scan( - &jobsdata.PipelineID, - &jobsdata.ID, - &jobsdata.Status, - &jobsdata.Stage, - &jobsdata.Name, - &jobsdata.Ref, - &jobsdata.CreatedAt, - &jobsdata.StartedAt, - &jobsdata.FinishedAt, - &jobsdata.Message, - &jobsdata.AuthorName, - ) - if err != nil { - return err - } - jobsdataarray = append(jobsdataarray, jobsdata) - pipelinedata.Jobs = jobsdataarray - } - datas.Dashboard = append(datas.Dashboard, pipelinedata) - } - err = pipelinerows.Err() - if err != nil { - return err - } - return nil -} - -func getTriggerPipelineid(jobURL, k8sVersion string) (string, error) { - url := jobURL + "/raw" - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", err - } - req.Close = true - req.Header.Set("Connection", "close") - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - data := string(body) - if data == "" { - return "0", nil - } - grep := exec.Command("grep", "-oP", "(?<="+k8sVersion+")[^ ]*") - ps := exec.Command("echo", data) - - // Get ps's stdout and attach it to grep's stdin. - pipe, _ := ps.StdoutPipe() - defer pipe.Close() - grep.Stdin = pipe - ps.Start() - - // Run and get the output of grep. - value, _ := grep.Output() - if string(value) == "" { - return "0", nil - } - result := strings.Split(string(value), "\"") - result = strings.Split(string(result[8]), "/") - if result[6] == "" { - return "0", nil - } - return result[6], nil -} - -// // jivaPipelineJobs will get pipeline jobs details from gitlab api -func pipelineJobsData(id int, token string, project string) (BuildJobs, error) { - url := jobURLGenerator(id, project) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - var obj BuildJobs - json.Unmarshal(body, &obj) - return obj, nil -} - -// pipelineData will fetch the data from gitlab API -func pipelineData(project, token string) (Pipeline, error) { - URL := pipelineURLGenerator(project) - req, err := http.NewRequest("GET", URL, nil) - if err != nil { - return nil, err - } - req.Close = true - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - var obj Pipeline - json.Unmarshal(data, &obj) - return obj, nil -} - -// genearete pipeline url according to project name -func pipelineURLGenerator(project string) string { - var projectID, Branch string - if project == "maya" { - projectID = MAYAID - Branch = MAYABRANCH - } else if project == "jiva" { - projectID = JIVAID - Branch = JIVABRANCH - } else if project == "istgt" { - projectID = ISTGTID - Branch = ISTGTBRANCH - } else if project == "zfs" { - projectID = ZFSID - Branch = ZFSBRANCH - } - generatedURL := BaseURL + "api/v4/projects/" + projectID + "/pipelines?ref=" + Branch - return generatedURL -} - -// genearete pipeline job url according to project name -func jobURLGenerator(id int, project string) string { - var projectID string - if project == "maya" { - projectID = MAYAID - } else if project == "jiva" { - projectID = JIVAID - } else if project == "istgt" { - projectID = ISTGTID - } else if project == "zfs" { - projectID = ZFSID - } - generatedURL := BaseURL + "api/v4/projects/" + projectID + "/pipelines/" + strconv.Itoa(id) + "/jobs" - return generatedURL -} - -// Generate joburl of baseline stage -func getBaselineJobWebURL(data BuildJobs) string { - var maxJobID = 0 - var jobURL string - for _, value := range data { - if value.Stage == "baseline" { - if value.ID > maxJobID { - maxJobID = value.ID - jobURL = value.WebURL - } - } - } - return jobURL -} diff --git a/handler/konvoy.go b/handler/konvoy.go index 8a2dd65..b2bef72 100644 --- a/handler/konvoy.go +++ b/handler/konvoy.go @@ -2,21 +2,17 @@ package handler import ( "encoding/json" - "fmt" - "io/ioutil" "net/http" - "strconv" "github.com/golang/glog" - "github.com/mayadata-io/ci-e2e-status/database" ) // KonvoyHandler return packet pipeline data to /packet path func KonvoyHandler(w http.ResponseWriter, r *http.Request) { // Allow cross origin request (w).Header().Set("Access-Control-Allow-Origin", "*") - datas := dashboard{} - err := QueryKonvoyData(&datas, "konvoy_pipeline", "konvoy_jobs") + datas := Openshiftdashboard{} + err := QueryData(&datas, "konvoy_pipeline", "konvoy_jobs") if err != nil { http.Error(w, err.Error(), 500) glog.Error(err) @@ -28,206 +24,3 @@ func KonvoyHandler(w http.ResponseWriter, r *http.Request) { } w.Write(out) } - -// QueryKonvoyData fetch the pipeline data as well as jobs data form Packet table of database -func QueryKonvoyData(datas *dashboard, pipelineTableName string, jobTableName string) error { - // Select all data from packetpipeline table of DB - query := fmt.Sprintf("SELECT id,sha,ref,status,web_url,kibana_url FROM %s ORDER BY build_pipelineid DESC;", pipelineTableName) - pipelinerows, err := database.Db.Query(query) - if err != nil { - return err - } - // Close DB connection after r/w operation - defer pipelinerows.Close() - // Iterate on each rows of pipeline table data for perform more operation related to pipeline Data - for pipelinerows.Next() { - pipelinedata := pipelineSummary{} - err = pipelinerows.Scan( - &pipelinedata.ID, - &pipelinedata.Sha, - &pipelinedata.Ref, - &pipelinedata.Status, - &pipelinedata.WebURL, - &pipelinedata.LogURL, - ) - if err != nil { - return err - } - // Query packetjobs data of respective pipeline using pipelineID from packetjobs table - jobsquery := fmt.Sprintf("SELECT * FROM %s WHERE pipelineid = $1 ORDER BY id;", jobTableName) - jobsrows, err := database.Db.Query(jobsquery, pipelinedata.ID) - if err != nil { - return err - } - // Close DB connection after r/w operation - defer jobsrows.Close() - jobsdataarray := []Jobssummary{} - // Iterate on each rows of table data for perform more operation related to pipelineJobsData - for jobsrows.Next() { - jobsdata := Jobssummary{} - err = jobsrows.Scan( - &jobsdata.PipelineID, - &jobsdata.ID, - &jobsdata.Status, - &jobsdata.Stage, - &jobsdata.Name, - &jobsdata.Ref, - &jobsdata.CreatedAt, - &jobsdata.StartedAt, - &jobsdata.FinishedAt, - &jobsdata.JobLogURL, - ) - if err != nil { - return err - } - // Append each row data to an array(jobsDataArray) - jobsdataarray = append(jobsdataarray, jobsdata) - // Add jobs details of pipeline into jobs field of pipelineData - pipelinedata.Jobs = jobsdataarray - } - // Append each pipeline data to datas of field Dashobard - datas.Dashboard = append(datas.Dashboard, pipelinedata) - } - err = pipelinerows.Err() - if err != nil { - return err - } - return nil -} - -// KonvoyData from gitlab api for Konvoy and dump to database -func KonvoyData(token, triggredIDColumnName, pipelineTableName, jobTableName string) { - query := fmt.Sprintf("SELECT id,%s FROM build_pipeline ORDER BY id DESC FETCH FIRST 20 ROWS ONLY;", triggredIDColumnName) - konvoyPipelineID, err := database.Db.Query(query) - if err != nil { - glog.Error("KONVOY pipeline quering data Error:", err) - return - } - for konvoyPipelineID.Next() { - var logURL string - pipelinedata := TriggredID{} - err = konvoyPipelineID.Scan( - &pipelinedata.BuildPID, - &pipelinedata.ID, - ) - defer konvoyPipelineID.Close() - konvoyPipelineData, err := konvoyPipeline(token, pipelinedata.ID) - if err != nil { - glog.Error(err) - return - } - pipelineJobsdata, err := konvoyPipelineJobs(konvoyPipelineData.ID, token) - if err != nil { - glog.Error(err) - return - } - if pipelinedata.ID != 0 && len(pipelineJobsdata) != 0 { - jobStartedAt := pipelineJobsdata[0].StartedAt - JobFinishedAt := pipelineJobsdata[len(pipelineJobsdata)-1].FinishedAt - logURL = Kibanaloglink(konvoyPipelineData.Sha, konvoyPipelineData.ID, konvoyPipelineData.Status, jobStartedAt, JobFinishedAt) - } - sqlStatement := fmt.Sprintf("INSERT INTO %s (build_pipelineid, id, sha, ref, status, web_url, kibana_url) VALUES ($1, $2, $3, $4, $5, $6, $7)"+ - "ON CONFLICT (build_pipelineid) DO UPDATE SET id = $2, sha = $3, ref = $4, status = $5, web_url = $6, kibana_url = $7 RETURNING id;", pipelineTableName) - id := 0 - err = database.Db.QueryRow(sqlStatement, - pipelinedata.BuildPID, - konvoyPipelineData.ID, - konvoyPipelineData.Sha, - konvoyPipelineData.Ref, - konvoyPipelineData.Status, - konvoyPipelineData.WebURL, - logURL, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for KONVOY Pipeline:", id) - if pipelinedata.ID != 0 { - for j := range pipelineJobsdata { - var jobLogURL string - sqlStatement := fmt.Sprintf("INSERT INTO %s (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, job_log_url) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)"+ - "ON CONFLICT (id) DO UPDATE SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9, job_log_url = $10 RETURNING id;", jobTableName) - id := 0 - if len(pipelineJobsdata) != 0 { - jobLogURL = Kibanaloglink(konvoyPipelineData.Sha, konvoyPipelineData.ID, konvoyPipelineData.Status, pipelineJobsdata[j].StartedAt, pipelineJobsdata[j].FinishedAt) - } - err = database.Db.QueryRow(sqlStatement, - konvoyPipelineData.ID, - pipelineJobsdata[j].ID, - pipelineJobsdata[j].Status, - pipelineJobsdata[j].Stage, - pipelineJobsdata[j].Name, - pipelineJobsdata[j].Ref, - pipelineJobsdata[j].CreatedAt, - pipelineJobsdata[j].StartedAt, - pipelineJobsdata[j].FinishedAt, - jobLogURL, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for KONVOY Jobs:", id) - } - } - } -} - -// konvoyPipeline will get data from gitlab api and store to DB -func konvoyPipeline(token string, pipelineID int) (*PlatformPipeline, error) { - dummyJSON := []byte(`{"id":0,"sha":"00000000000000000000","ref":"none","status":"none","web_url":"none"}`) - if pipelineID == 0 { - var obj PlatformPipeline - json.Unmarshal(dummyJSON, &obj) - return &obj, nil - } - // Store packet pipeline data form gitlab api to packetObj - url := BaseURL + "api/v4/projects/" + KONVOYID + "/pipelines/" + strconv.Itoa(pipelineID) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - // Set header for api request - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - // Unmarshal response data - var obj PlatformPipeline - json.Unmarshal(body, &obj) - if obj.ID == 0 { - return nil, fmt.Errorf("Pipeline data not found") - } - return &obj, nil -} - -// konvoyPipelineJobs will get pipeline jobs details from gitlab jobs api -func konvoyPipelineJobs(pipelineID int, token string) (Jobs, error) { - // Generate pipeline jobs api url using BaseURL, pipelineID and KONVOYID - if pipelineID == 0 { - return nil, nil - } - url := BaseURL + "api/v4/projects/" + KONVOYID + "/pipelines/" + strconv.Itoa(pipelineID) + "/jobs?per_page=100" - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - // Set header for api request - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - // Unmarshal response data - var obj Jobs - json.Unmarshal(body, &obj) - return obj, nil -} diff --git a/handler/opeshift-release.go b/handler/opeshift-release.go index 0b6e6ad..61954d2 100644 --- a/handler/opeshift-release.go +++ b/handler/opeshift-release.go @@ -2,24 +2,17 @@ package handler import ( "encoding/json" - "fmt" - "io/ioutil" "net/http" - "regexp" - "strconv" - "strings" - "time" "github.com/golang/glog" - "github.com/mayadata-io/ci-e2e-status/database" ) -// OpenshiftHandlerRelease return eks pipeline data to /build path -func OpenshiftHandlerRelease(w http.ResponseWriter, r *http.Request) { +// OpenshiftHandlerReleasee return eks pipeline data to /build path +func OpenshiftHandlerReleasee(w http.ResponseWriter, r *http.Request) { // Allow cross origin request (w).Header().Set("Access-Control-Allow-Origin", "*") datas := Openshiftdashboard{} - err := QueryReleasePipelineData(&datas, "release_pipeline_data", "release_jobs_data") + err := QueryData(&datas, "release_pipeline_data", "release_jobs_data") if err != nil { http.Error(w, err.Error(), 500) glog.Error(err) @@ -33,229 +26,3 @@ func OpenshiftHandlerRelease(w http.ResponseWriter, r *http.Request) { } w.Write(out) } - -// QueryReleasePipelineData fetches the builddashboard data from the db -func QueryReleasePipelineData(datas *Openshiftdashboard, pipelineTable string, jobsTable string) error { - pipelineQuery := fmt.Sprintf("SELECT * FROM %s ORDER BY id DESC;", pipelineTable) - pipelinerows, err := database.Db.Query(pipelineQuery) - if err != nil { - return err - } - defer pipelinerows.Close() - for pipelinerows.Next() { - pipelinedata := OpenshiftpipelineSummary{} - err = pipelinerows.Scan( - &pipelinedata.Project, - &pipelinedata.ID, - &pipelinedata.Sha, - &pipelinedata.Ref, - &pipelinedata.Status, - &pipelinedata.WebURL, - &pipelinedata.OpenshiftPID, - &pipelinedata.LogURL, - &pipelinedata.ReleaseTag, - ) - if err != nil { - return err - } - jobsquery := fmt.Sprintf("SELECT pipelineid, id, status , stage , name , ref , created_at , started_at , finished_at FROM %s WHERE pipelineid = $1 ORDER BY id;", jobsTable) - jobsrows, err := database.Db.Query(jobsquery, pipelinedata.ID) - if err != nil { - return err - } - defer jobsrows.Close() - jobsdataarray := []BuildJobssummary{} - for jobsrows.Next() { - jobsdata := BuildJobssummary{} - err = jobsrows.Scan( - &jobsdata.PipelineID, - &jobsdata.ID, - &jobsdata.Status, - &jobsdata.Stage, - &jobsdata.Name, - &jobsdata.Ref, - &jobsdata.CreatedAt, - &jobsdata.StartedAt, - &jobsdata.FinishedAt, - ) - if err != nil { - return err - } - jobsdataarray = append(jobsdataarray, jobsdata) - pipelinedata.Jobs = jobsdataarray - } - datas.Dashboard = append(datas.Dashboard, pipelinedata) - } - err = pipelinerows.Err() - if err != nil { - return err - } - return nil -} - -func getReleaseData(token, branch string) (Pipeline, error) { - URL := "https://gitlab.openebs.ci/api/v4/projects/36/pipelines?ref=release-branch" - req, err := http.NewRequest("GET", URL, nil) - if err != nil { - return nil, err - } - req.Close = true - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - var obj Pipeline - json.Unmarshal(data, &obj) - return obj, nil -} - -func releasePipelineJobs(pipelineID int, token string) (Jobs, error) { - // Generate pipeline jobs api url using BaseURL, pipelineID and OPENSHIFTID - urlTmp := BaseURL + "api/v4/projects/36/pipelines/" + strconv.Itoa(pipelineID) + "/jobs?page=" - var obj Jobs - for i := 1; ; i++ { - url := urlTmp + strconv.Itoa(i) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - // Set header for api request - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - client := http.Client{ - Timeout: time.Minute * time.Duration(2), - } - res, err := client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - if string(body) == "[]" { - break - } - var tmpObj Jobs - err = json.Unmarshal(body, &tmpObj) - glog.Infoln("error ", err) - obj = append(obj, tmpObj...) - } - return obj, nil -} - -// openshiftCommit from gitlab api and store to database -func releaseBranch(token, project, branch, pipelineTable, jobTable string) { - var logURL string - var releaseTag string - releaseData, err := getReleaseData(token, branch) - if err != nil { - glog.Error(err) - return - } - for i := range releaseData { - pipelineJobsData, err := releasePipelineJobs(releaseData[i].ID, token) - if err != nil { - glog.Error(err) - return - } - glog.Infoln("pipelieID :-> " + strconv.Itoa(releaseData[i].ID) + " || JobSLegth :-> " + strconv.Itoa(len(pipelineJobsData))) - if len(pipelineJobsData) != 0 { - jobStartedAt := pipelineJobsData[0].StartedAt - JobFinishedAt := pipelineJobsData[len(pipelineJobsData)-1].FinishedAt - logURL = Kibanaloglink(releaseData[i].Sha, releaseData[i].ID, releaseData[i].Status, jobStartedAt, JobFinishedAt) - } - releaseTag, err = getReleaseTag(pipelineJobsData, token) - if err != nil { - glog.Error(err) - } - - glog.Infoln("releaseTagFuc Result : - > : ", releaseTag) - // Add pipelines data to Database - sqlStatement := fmt.Sprintf("INSERT INTO %s (project, id, sha, ref, status, web_url, openshift_pid, kibana_url, release_tag) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"+ - "ON CONFLICT (id) DO UPDATE SET status = $5, openshift_pid = $7, kibana_url = $8, release_tag = $9 RETURNING id;", pipelineTable) - id := 0 - err = database.Db.QueryRow(sqlStatement, - project, - releaseData[i].ID, - releaseData[i].Sha, - releaseData[i].Ref, - releaseData[i].Status, - releaseData[i].WebURL, - releaseData[i].ID, - logURL, - releaseTag, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infof("New record ID for %s Pipeline: %d", project, id) - - // Add pipeline jobs data to Database - for j := range pipelineJobsData { - sqlStatement := fmt.Sprintf("INSERT INTO %s (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"+ - "ON CONFLICT (id) DO UPDATE SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 RETURNING id;", jobTable) - id := 0 - err = database.Db.QueryRow(sqlStatement, - releaseData[i].ID, - pipelineJobsData[j].ID, - pipelineJobsData[j].Status, - pipelineJobsData[j].Stage, - pipelineJobsData[j].Name, - pipelineJobsData[j].Ref, - pipelineJobsData[j].CreatedAt, - pipelineJobsData[j].StartedAt, - pipelineJobsData[j].FinishedAt, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infof("New record ID for %s pipeline Jobs: %d", project, id) - } - } -} - -func getReleaseTag(jobsData Jobs, token string) (string, error) { - var jobURL string - for _, value := range jobsData { - if value.Name == "K9YC-OpenEBS" { - jobURL = value.WebURL + "/raw" - } - } - req, err := http.NewRequest("GET", jobURL, nil) - if err != nil { - return "NA", err - } - req.Close = true - req.Header.Set("Connection", "close") - client := http.Client{ - Timeout: time.Minute * time.Duration(1), - } - res, err := client.Do(req) - if err != nil { - return "NA", err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - data := string(body) - if data == "" { - return "NA", err - } - re := regexp.MustCompile("releaseTag[^ ]*") - value := re.FindString(data) - result := strings.Split(string(value), "=") - if result != nil && len(result) > 1 { - if result[1] == "" { - return "NA", nil - } - releaseVersion := strings.Split(result[1], "\n") - return releaseVersion[0], nil - } - return "NA", nil -} diff --git a/handler/packet.go b/handler/packet.go index 6533b2f..b163fd1 100644 --- a/handler/packet.go +++ b/handler/packet.go @@ -2,21 +2,17 @@ package handler import ( "encoding/json" - "fmt" - "io/ioutil" "net/http" - "strconv" "github.com/golang/glog" - "github.com/mayadata-io/ci-e2e-status/database" ) -// PacketHandlerV15 return packet pipeline data to /packet path -func PacketHandlerV15(w http.ResponseWriter, r *http.Request) { +// PacketHandlerAntepenultimate return packet pipeline data to /packet path +func PacketHandlerAntepenultimate(w http.ResponseWriter, r *http.Request) { // Allow cross origin request (w).Header().Set("Access-Control-Allow-Origin", "*") - datas := dashboard{} - err := QueryPacketData(&datas, "packet_pipeline_v15", "packet_jobs_v15") + datas := Openshiftdashboard{} + err := QueryData(&datas, "packet_pipeline_k8s_antepenultimate", "packet_jobs_k8s_antepenultimate") if err != nil { http.Error(w, err.Error(), 500) glog.Error(err) @@ -29,13 +25,13 @@ func PacketHandlerV15(w http.ResponseWriter, r *http.Request) { w.Write(out) } -// PacketHandlerV14 return packet pipeline data to /packet path +// PacketHandlerPenultimate return packet pipeline data to /packet path // TODO -func PacketHandlerV14(w http.ResponseWriter, r *http.Request) { +func PacketHandlerPenultimate(w http.ResponseWriter, r *http.Request) { // Allow cross origin request (w).Header().Set("Access-Control-Allow-Origin", "*") - datas := dashboard{} - err := QueryPacketData(&datas, "packet_pipeline_v14", "packet_jobs_v14") + datas := Openshiftdashboard{} + err := QueryData(&datas, "packet_pipeline_k8s_penultimate", "packet_jobs_k8s_penultimate") if err != nil { http.Error(w, err.Error(), 500) glog.Error(err) @@ -48,13 +44,13 @@ func PacketHandlerV14(w http.ResponseWriter, r *http.Request) { w.Write(out) } -// PacketHandlerV13 return packet pipeline data to /packet path +// PacketHandlerUltimate return packet pipeline data to /packet path // TODO -func PacketHandlerV13(w http.ResponseWriter, r *http.Request) { +func PacketHandlerUltimate(w http.ResponseWriter, r *http.Request) { // Allow cross origin request (w).Header().Set("Access-Control-Allow-Origin", "*") - datas := dashboard{} - err := QueryPacketData(&datas, "packet_pipeline_v13", "packet_jobs_v13") + datas := Openshiftdashboard{} + err := QueryData(&datas, "packet_pipeline_k8s_ultimate", "packet_jobs_k8s_ultimate") if err != nil { http.Error(w, err.Error(), 500) glog.Error(err) @@ -66,206 +62,3 @@ func PacketHandlerV13(w http.ResponseWriter, r *http.Request) { } w.Write(out) } - -// PacketData from gitlab api for Packet and dump to database -func PacketData(token, triggredIDColumnName, pipelineTableName, jobTableName string) { - query := fmt.Sprintf("SELECT id,%s FROM build_pipeline ORDER BY id DESC FETCH FIRST 20 ROWS ONLY;", triggredIDColumnName) - packetPipelineID, err := database.Db.Query(query) - if err != nil { - glog.Error("PACKET pipeline quering data Error:", err) - return - } - for packetPipelineID.Next() { - var logURL string - pipelinedata := TriggredID{} - err = packetPipelineID.Scan( - &pipelinedata.BuildPID, - &pipelinedata.ID, - ) - defer packetPipelineID.Close() - packetPipelineData, err := packetPipeline(token, pipelinedata.ID) - if err != nil { - glog.Error(err) - return - } - pipelineJobsdata, err := packetPipelineJobs(packetPipelineData.ID, token) - if err != nil { - glog.Error(err) - return - } - if pipelinedata.ID != 0 && len(pipelineJobsdata) != 0 { - jobStartedAt := pipelineJobsdata[0].StartedAt - JobFinishedAt := pipelineJobsdata[len(pipelineJobsdata)-1].FinishedAt - logURL = Kibanaloglink(packetPipelineData.Sha, packetPipelineData.ID, packetPipelineData.Status, jobStartedAt, JobFinishedAt) - } - sqlStatement := fmt.Sprintf("INSERT INTO %s (build_pipelineid, id, sha, ref, status, web_url, kibana_url) VALUES ($1, $2, $3, $4, $5, $6, $7)"+ - "ON CONFLICT (build_pipelineid) DO UPDATE SET id = $2, sha = $3, ref = $4, status = $5, web_url = $6, kibana_url = $7 RETURNING id;", pipelineTableName) - id := 0 - err = database.Db.QueryRow(sqlStatement, - pipelinedata.BuildPID, - packetPipelineData.ID, - packetPipelineData.Sha, - packetPipelineData.Ref, - packetPipelineData.Status, - packetPipelineData.WebURL, - logURL, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for PACKET Pipeline:", id) - if pipelinedata.ID != 0 { - for j := range pipelineJobsdata { - var jobLogURL string - sqlStatement := fmt.Sprintf("INSERT INTO %s (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at, job_log_url) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)"+ - "ON CONFLICT (id) DO UPDATE SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9, job_log_url = $10 RETURNING id;", jobTableName) - id := 0 - if len(pipelineJobsdata) != 0 { - jobLogURL = Kibanaloglink(packetPipelineData.Sha, packetPipelineData.ID, packetPipelineData.Status, pipelineJobsdata[j].StartedAt, pipelineJobsdata[j].FinishedAt) - } - err = database.Db.QueryRow(sqlStatement, - packetPipelineData.ID, - pipelineJobsdata[j].ID, - pipelineJobsdata[j].Status, - pipelineJobsdata[j].Stage, - pipelineJobsdata[j].Name, - pipelineJobsdata[j].Ref, - pipelineJobsdata[j].CreatedAt, - pipelineJobsdata[j].StartedAt, - pipelineJobsdata[j].FinishedAt, - jobLogURL, - ).Scan(&id) - if err != nil { - glog.Error(err) - } - glog.Infoln("New record ID for PACKET Jobs:", id) - } - } - } -} - -// packetPipeline will get data from gitlab api and store to DB -func packetPipeline(token string, pipelineID int) (*PlatformPipeline, error) { - dummyJSON := []byte(`{"id":0,"sha":"00000000000000000000","ref":"none","status":"none","web_url":"none"}`) - if pipelineID == 0 { - var obj PlatformPipeline - json.Unmarshal(dummyJSON, &obj) - return &obj, nil - } - // Store packet pipeline data form gitlab api to packetObj - url := BaseURL + "api/v4/projects/" + PACKETID + "/pipelines/" + strconv.Itoa(pipelineID) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - // Set header for api request - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - // Unmarshal response data - var obj PlatformPipeline - json.Unmarshal(body, &obj) - if obj.ID == 0 { - return nil, fmt.Errorf("Pipeline data not found") - } - return &obj, nil -} - -// packetPipelineJobs will get pipeline jobs details from gitlab jobs api -func packetPipelineJobs(pipelineID int, token string) (Jobs, error) { - // Generate pipeline jobs api url using BaseURL, pipelineID and PACKETID - if pipelineID == 0 { - return nil, nil - } - url := BaseURL + "api/v4/projects/" + PACKETID + "/pipelines/" + strconv.Itoa(pipelineID) + "/jobs?per_page=100" - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Close = true - // Set header for api request - req.Header.Set("Connection", "close") - req.Header.Add("PRIVATE-TOKEN", token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) - // Unmarshal response data - var obj Jobs - json.Unmarshal(body, &obj) - return obj, nil -} - -// QueryPacketData fetch the pipeline data as well as jobs data form Packet table of database -func QueryPacketData(datas *dashboard, pipelineTableName string, jobTableName string) error { - // Select all data from packetpipeline table of DB - query := fmt.Sprintf("SELECT id,sha,ref,status,web_url,kibana_url FROM %s ORDER BY build_pipelineid DESC;", pipelineTableName) - pipelinerows, err := database.Db.Query(query) - if err != nil { - return err - } - // Close DB connection after r/w operation - defer pipelinerows.Close() - // Iterate on each rows of pipeline table data for perform more operation related to pipeline Data - for pipelinerows.Next() { - pipelinedata := pipelineSummary{} - err = pipelinerows.Scan( - &pipelinedata.ID, - &pipelinedata.Sha, - &pipelinedata.Ref, - &pipelinedata.Status, - &pipelinedata.WebURL, - &pipelinedata.LogURL, - ) - if err != nil { - return err - } - // Query packetjobs data of respective pipeline using pipelineID from packetjobs table - jobsquery := fmt.Sprintf("SELECT * FROM %s WHERE pipelineid = $1 ORDER BY id;", jobTableName) - jobsrows, err := database.Db.Query(jobsquery, pipelinedata.ID) - if err != nil { - return err - } - // Close DB connection after r/w operation - defer jobsrows.Close() - jobsdataarray := []Jobssummary{} - // Iterate on each rows of table data for perform more operation related to pipelineJobsData - for jobsrows.Next() { - jobsdata := Jobssummary{} - err = jobsrows.Scan( - &jobsdata.PipelineID, - &jobsdata.ID, - &jobsdata.Status, - &jobsdata.Stage, - &jobsdata.Name, - &jobsdata.Ref, - &jobsdata.CreatedAt, - &jobsdata.StartedAt, - &jobsdata.FinishedAt, - &jobsdata.JobLogURL, - ) - if err != nil { - return err - } - // Append each row data to an array(jobsDataArray) - jobsdataarray = append(jobsdataarray, jobsdata) - // Add jobs details of pipeline into jobs field of pipelineData - pipelinedata.Jobs = jobsdataarray - } - // Append each pipeline data to datas of field Dashobard - datas.Dashboard = append(datas.Dashboard, pipelinedata) - } - err = pipelinerows.Err() - if err != nil { - return err - } - return nil -} diff --git a/handler/trigger-data.go b/handler/trigger-data.go new file mode 100644 index 0000000..336cbc0 --- /dev/null +++ b/handler/trigger-data.go @@ -0,0 +1,241 @@ +package handler + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "github.com/mayadata-io/ci-e2e-status/database" +) + +// QueryData fetches the builddashboard data from the db +func QueryData(datas *Openshiftdashboard, pipelineTable string, jobsTable string) error { + pipelineQuery := fmt.Sprintf("SELECT * FROM %s ORDER BY id DESC;", pipelineTable) + pipelinerows, err := database.Db.Query(pipelineQuery) + if err != nil { + return err + } + defer pipelinerows.Close() + for pipelinerows.Next() { + pipelinedata := OpenshiftpipelineSummary{} + err = pipelinerows.Scan( + &pipelinedata.Project, + &pipelinedata.ID, + &pipelinedata.Sha, + &pipelinedata.Ref, + &pipelinedata.Status, + &pipelinedata.WebURL, + &pipelinedata.OpenshiftPID, + &pipelinedata.LogURL, + &pipelinedata.ReleaseTag, + ) + if err != nil { + return err + } + jobsquery := fmt.Sprintf("SELECT pipelineid, id, status , stage , name , ref , created_at , started_at , finished_at FROM %s WHERE pipelineid = $1 ORDER BY id;", jobsTable) + jobsrows, err := database.Db.Query(jobsquery, pipelinedata.ID) + if err != nil { + return err + } + defer jobsrows.Close() + jobsdataarray := []BuildJobssummary{} + for jobsrows.Next() { + jobsdata := BuildJobssummary{} + err = jobsrows.Scan( + &jobsdata.PipelineID, + &jobsdata.ID, + &jobsdata.Status, + &jobsdata.Stage, + &jobsdata.Name, + &jobsdata.Ref, + &jobsdata.CreatedAt, + &jobsdata.StartedAt, + &jobsdata.FinishedAt, + ) + if err != nil { + return err + } + jobsdataarray = append(jobsdataarray, jobsdata) + pipelinedata.Jobs = jobsdataarray + } + datas.Dashboard = append(datas.Dashboard, pipelinedata) + } + err = pipelinerows.Err() + if err != nil { + return err + } + return nil +} + +func getPipelineData(token, project, branch string) (Pipeline, error) { + URL := "https://gitlab.openebs.ci/api/v4/projects/" + project + "/pipelines?ref=" + branch + req, err := http.NewRequest("GET", URL, nil) + if err != nil { + return nil, err + } + req.Close = true + req.Header.Set("Connection", "close") + req.Header.Add("PRIVATE-TOKEN", token) + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + var obj Pipeline + json.Unmarshal(data, &obj) + return obj, nil +} + +func releasePipelineJobs(pipelineID int, token string, project string) (Jobs, error) { + // Generate pipeline jobs api url using BaseURL, pipelineID and OPENSHIFTID + urlTmp := BaseURL + "api/v4/projects/" + project + "/pipelines/" + strconv.Itoa(pipelineID) + "/jobs?page=" + var obj Jobs + for i := 1; ; i++ { + url := urlTmp + strconv.Itoa(i) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Close = true + // Set header for api request + req.Header.Set("Connection", "close") + req.Header.Add("PRIVATE-TOKEN", token) + client := http.Client{ + Timeout: time.Minute * time.Duration(2), + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + if string(body) == "[]" { + break + } + var tmpObj Jobs + err = json.Unmarshal(body, &tmpObj) + glog.Infoln("error ", err) + obj = append(obj, tmpObj...) + } + return obj, nil +} + +// openshiftCommit from gitlab api and store to database +func getPlatformData(token, project, branch, pipelineTable, jobTable string) { + var logURL string + var imageTag string + pipelineData, err := getPipelineData(token, project, branch) + if err != nil { + glog.Error(err) + return + } + for i := range pipelineData { + pipelineJobsData, err := releasePipelineJobs(pipelineData[i].ID, token, project) + if err != nil { + glog.Error(err) + return + } + glog.Infoln("pipelieID :-> " + strconv.Itoa(pipelineData[i].ID) + " || JobSLegth :-> " + strconv.Itoa(len(pipelineJobsData))) + if len(pipelineJobsData) != 0 { + jobStartedAt := pipelineJobsData[0].StartedAt + JobFinishedAt := pipelineJobsData[len(pipelineJobsData)-1].FinishedAt + logURL = Kibanaloglink(pipelineData[i].Sha, pipelineData[i].ID, pipelineData[i].Status, jobStartedAt, JobFinishedAt) + } + imageTag, err = getImageTag(pipelineJobsData, token) + if err != nil { + glog.Error(err) + } + + glog.Infoln("releaseTagFuc Result : - > : ", imageTag) + // Add pipelines data to Database + sqlStatement := fmt.Sprintf("INSERT INTO %s (project, id, sha, ref, status, web_url, openshift_pid, kibana_url, release_tag) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"+ + "ON CONFLICT (id) DO UPDATE SET status = $5, openshift_pid = $7, kibana_url = $8, release_tag = $9 RETURNING id;", pipelineTable) + id := 0 + err = database.Db.QueryRow(sqlStatement, + project, + pipelineData[i].ID, + pipelineData[i].Sha, + pipelineData[i].Ref, + pipelineData[i].Status, + pipelineData[i].WebURL, + pipelineData[i].ID, + logURL, + imageTag, + ).Scan(&id) + if err != nil { + glog.Error(err) + } + glog.Infof("New record ID for %s Pipeline: %d", project, id) + + // Add pipeline jobs data to Database + for j := range pipelineJobsData { + sqlStatement := fmt.Sprintf("INSERT INTO %s (pipelineid, id, status, stage, name, ref, created_at, started_at, finished_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"+ + "ON CONFLICT (id) DO UPDATE SET status = $3, stage = $4, name = $5, ref = $6, created_at = $7, started_at = $8, finished_at = $9 RETURNING id;", jobTable) + id := 0 + err = database.Db.QueryRow(sqlStatement, + pipelineData[i].ID, + pipelineJobsData[j].ID, + pipelineJobsData[j].Status, + pipelineJobsData[j].Stage, + pipelineJobsData[j].Name, + pipelineJobsData[j].Ref, + pipelineJobsData[j].CreatedAt, + pipelineJobsData[j].StartedAt, + pipelineJobsData[j].FinishedAt, + ).Scan(&id) + if err != nil { + glog.Error(err) + } + glog.Infof("New record ID for %s pipeline Jobs: %d", project, id) + } + } +} + +func getImageTag(jobsData Jobs, token string) (string, error) { + var jobURL string + for _, value := range jobsData { + if value.Name == "K9YC-OpenEBS" || value.Name == "openebs-deploy" || value.Name == "XJGT-OPENEBS-KONVOY-DEPLOY" { + jobURL = value.WebURL + "/raw" + } + } + req, err := http.NewRequest("GET", jobURL, nil) + if err != nil { + return "NA", err + } + req.Close = true + req.Header.Set("Connection", "close") + client := http.Client{ + Timeout: time.Minute * time.Duration(1), + } + res, err := client.Do(req) + if err != nil { + return "NA", err + } + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + data := string(body) + if data == "" { + return "NA", err + } + re := regexp.MustCompile("releaseTag[^ ]*") + value := re.FindString(data) + result := strings.Split(string(value), "=") + if result != nil && len(result) > 1 { + if result[1] == "" { + return "NA", nil + } + releaseVersion := strings.Split(result[1], "\n") + return releaseVersion[0], nil + } + return "NA", nil +} diff --git a/handler/types.go b/handler/types.go index 4b756ee..b856b59 100644 --- a/handler/types.go +++ b/handler/types.go @@ -2,12 +2,13 @@ package handler // PlatformID assignment from gitlab repository const ( - PACKETID = "27" - KONVOYID = "34" - MAYAID = "7" - JIVAID = "6" - ISTGTID = "5" - ZFSID = "8" + PACKETID = "27" + KONVOYID = "34" + OPENSHIFTID = "36" + MAYAID = "7" + JIVAID = "6" + ISTGTID = "5" + ZFSID = "8" ) // BranchName assignment from gitlab repository @@ -125,8 +126,6 @@ type BuildJobssummary struct { CreatedAt string `json:"created_at"` StartedAt string `json:"started_at"` FinishedAt string `json:"finished_at"` - Message string `json:"message"` - AuthorName string `json:"author_name"` } // BuildpipelineSummary contains the details of a builds pipelines diff --git a/handler/updatedatabase.go b/handler/updatedatabase.go index 06813f9..b0e5ab1 100644 --- a/handler/updatedatabase.go +++ b/handler/updatedatabase.go @@ -16,29 +16,28 @@ func UpdateDatabase() { glog.Fatalf("TOKEN environment variable required") } // Update the database, This wil run only first time - BuildData(token) - k8sVersion := []string{"v15", "v14", "v13"} + k8sVersion := []string{"ultimate", "penultimate", "antepenultimate"} for _, k8sVersion := range k8sVersion { - columnName := "packet_" + k8sVersion + "_pid" - pipelineTable := "packet_pipeline_" + k8sVersion - jobTable := "packet_jobs_" + k8sVersion - go PacketData(token, columnName, pipelineTable, jobTable) + branch := "k8s-" + k8sVersion + pipelineTable := "packet_pipeline_k8s_" + k8sVersion + jobTable := "packet_jobs_k8s_" + k8sVersion + getPlatformData(token, PACKETID, branch, pipelineTable, jobTable) //e2e-packet } - go KonvoyData(token, "konvoy_pid", "konvoy_pipeline", "konvoy_jobs") - go releaseBranch(token, "e2e-openshift", "release-branch", "release_pipeline_data", "release_jobs_data") + go getPlatformData(token, KONVOYID, "release-branch", "konvoy_pipeline", "konvoy_jobs") // e2e-konvoy + go getPlatformData(token, OPENSHIFTID, "release-branch", "release_pipeline_data", "release_jobs_data") //e2e-openshift // loop will iterate at every 2nd minute and update the database tick := time.Tick(2 * time.Minute) for range tick { - BuildData(token) + k8sVersion := []string{"ultimate", "penultimate", "antepenultimate"} for _, k8sVersion := range k8sVersion { - columnName := "packet_" + k8sVersion + "_pid" - pipelineTable := "packet_pipeline_" + k8sVersion - jobTable := "packet_jobs_" + k8sVersion - go PacketData(token, columnName, pipelineTable, jobTable) + branch := "k8s-" + k8sVersion + pipelineTable := "packet_pipeline_k8s_" + k8sVersion + jobTable := "packet_jobs_k8s_" + k8sVersion + getPlatformData(token, PACKETID, branch, pipelineTable, jobTable) //e2e-packet } - go KonvoyData(token, "konvoy_pid", "konvoy_pipeline", "konvoy_jobs") - go releaseBranch(token, "e2e-openshift", "release-branch", "release_pipeline_data", "release_jobs_data") - + go getPlatformData(token, KONVOYID, "release-branch", "konvoy_pipeline", "konvoy_jobs") // e2e-konvoy + go getPlatformData(token, OPENSHIFTID, "release-branch", "release_pipeline_data", "release_jobs_data") //e2e-openshift } + } diff --git a/main.go b/main.go index 9877ca6..6d83406 100644 --- a/main.go +++ b/main.go @@ -16,12 +16,11 @@ func main() { // Initailze Db connection database.InitDb() // Return value to all / api path - http.HandleFunc("/packet/v15", handler.PacketHandlerV15) - http.HandleFunc("/packet/v14", handler.PacketHandlerV14) - http.HandleFunc("/packet/v13", handler.PacketHandlerV13) - http.HandleFunc("/build", handler.Buildhandler) + http.HandleFunc("/packet/ultimate", handler.PacketHandlerUltimate) + http.HandleFunc("/packet/penultimate", handler.PacketHandlerPenultimate) + http.HandleFunc("/packet/antepenultimate", handler.PacketHandlerAntepenultimate) http.HandleFunc("/konvoy", handler.KonvoyHandler) - http.HandleFunc("/openshift/release", handler.OpenshiftHandlerRelease) + http.HandleFunc("/openshift/release", handler.OpenshiftHandlerReleasee) glog.Infof("Listening on http://0.0.0.0:3000")