Libtrack 24/add npm dependency graph

exe/analyze

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 require "library_version_analysis"
-results = LibraryVersionAnalysis::CheckVersionStatus.run(spreadsheet_id: ARGV[0], online: ARGV[1], online_node: ARGV[2], mobile: ARGV[3])
+results = LibraryVersionAnalysis::CheckVersionStatus.run(repository: ARGV[0])
 puts JSON.generate(results)

lib/library_version_analysis/analyze.rb

@@ -1,8 +1,7 @@
 module LibraryVersionAnalysis
   class Analyze
-    def self.go
-      spreadsheet_id = ENV["VERSION_STATUS_SPREADSHEET_ID"]
-      results = LibraryVersionAnalysis::CheckVersionStatus.run(spreadsheet_id: spreadsheet_id, online: "true", online_node: "true", mobile: "false")
+    def self.go(repository)
+      results = LibraryVersionAnalysis::CheckVersionStatus.run(repository: repository)
 
       merged_result = {}
       results.keys.each { |key| merged_result.merge!(results[key]) }

lib/library_version_analysis/check_version_status.rb

@@ -2,6 +2,8 @@
 require "google/apis/sheets_v4"
 require "open3"
 require "pry"
+require "uri"
+require "net/https"
 
 module LibraryVersionAnalysis
   Versionline = Struct.new(
@@ -16,29 +18,51 @@ module LibraryVersionAnalysis
     :minor,
     :patch,
     :age,
+    :source,
+    :dependency_graph,
     :dependabot_created_at,
     :dependabot_permalink,
     keyword_init: true
   )
   MetaData = Struct.new(:total_age, :total_releases, :total_major, :total_minor, :total_patch, :total_cvss)
   ModeSummary = Struct.new(:one_major, :two_major, :three_plus_major, :minor, :patch, :total, :total_lib_years, :total_cvss, :unowned_issues, :one_number)
 
+  LibNode = Struct.new(
+    :name,
+    :parents,
+    keyword_init: true
+  ) do |new_class|
+    def deep_to_h
+      h = {}
+      h[:name] = name
+      h[:parents] = parents&.map(&:deep_to_h)
+      h
+    end
+  end
+
   DEV_OUTPUT = true
 
   class CheckVersionStatus
-    def self.run(spreadsheet_id:, online: "true", online_node: "true", mobile: "true")
+    def self.run(repository:)
       c = CheckVersionStatus.new
-      mode_results = c.go(spreadsheet_id, online == "true", online_node == "true", mobile == "true")
+      mode_results = c.go(repository: repository)
 
       return c.build_mode_results(mode_results)
     end
 
-    def go(spreadsheet_id, online, online_node, mobile)
+    def go(repository:)
       puts "Check Version" if DEV_OUTPUT
 
-      meta_data_online_node, mode_online_node = go_online_node(spreadsheet_id) if online_node
-      meta_data_online, mode_online = go_online(spreadsheet_id) if online
-      meta_data_mobile, mode_mobile = go_mobile(spreadsheet_id) if mobile
+      # TODO: once we fully parameterize this, these go away
+      online = false
+      online_node = true
+      # online = true
+      # online_node = false
+      mobile = false
+
+      meta_data_online_node, mode_online_node = go_online_node(repository) if online_node
+      meta_data_online, mode_online = go_online(repository) if online
+      meta_data_mobile, mode_mobile = go_mobile(repository) if mobile
 
       print_summary("online", meta_data_online, mode_online) if online && DEV_OUTPUT
       print_summary("online_node", meta_data_online_node, mode_online_node) if online_node && DEV_OUTPUT
@@ -53,10 +77,10 @@ def go(spreadsheet_id, online, online_node, mobile)
       }
     end
 
-    def go_online(spreadsheet_id)
+    def go_online(repository)
       puts "  online" if DEV_OUTPUT
-      online = Online.new
-      meta_data_online, mode_online = get_version_summary(online, "OnlineVersionData!A:Q", spreadsheet_id, "ONLINE")
+      online = Online.new("Jobber")
+      meta_data_online, mode_online = get_version_summary(online, "OnlineVersionData!A:Q", repository, "ONLINE")
 
       return meta_data_online, mode_online
     end
@@ -77,14 +101,14 @@ def go_mobile(spreadsheet_id)
       return meta_data_mobile, mode_mobile
     end
 
-    def get_version_summary(parser, range, spreadsheet_id, source)
+    def get_version_summary(parser, range, repository, source)
       parsed_results, meta_data = parser.get_versions
 
       mode = get_mode_summary(parsed_results, meta_data)
-      data = spreadsheet_data(parsed_results, source)
+      data = server_data(parsed_results, repository)
 
-      puts "    updating spreadsheet" if DEV_OUTPUT
-      update_spreadsheet(spreadsheet_id, range, data)
+      puts "    updating server" if DEV_OUTPUT
+      update_server(data)
 
       puts "    slack notify" if DEV_OUTPUT
       notify(parsed_results)
@@ -97,49 +121,91 @@ def one_number(mode_summary)
       return mode_summary.three_plus_major * 50 + mode_summary.two_major * 20 + mode_summary.one_major * 10 + mode_summary.minor + mode_summary.patch * 0.5
     end
 
-    def spreadsheet_data(results, source)
-      header_row = %w(name owner parent source current_version current_version_date latest_version latest_version_date major minor patch age cve note cve_label cve_severity note_lookup_key)
-      data = [header_row]
-
-      data << ["Updated: #{Time.now.utc}"]
+    def server_data(results, repository)
+      libraries = []
+      new_versions = []
+      vulns = []
+      dependencies = []
 
+      missing_dependency_keys = [] # TODO: handle missing keys
       results.each do |name, row|
-        data << [
-          name,
-          row.owner,
-          row.parent,
-          source,
-          row.current_version,
-          row.current_version_date,
-          row.latest_version,
-          row.latest_version_date,
-          row.major,
-          row.minor,
-          row.patch,
-          row.age,
-          row.cvss,
-          '=IFERROR(concatenate(vlookup(indirect("Q" & row()),Notes!A:E,4,false), ":", concatenate(vlookup(indirect("Q" & row()),Notes!A:E,5,false))))',
-          '=IFERROR(vlookup(indirect("Q" & row()),Notes!A:E,4,false), IFERROR(trim(LEFT(INDIRECT("Q" & row()), SEARCH("[", INDIRECT("M" & row()))-1))))',
-          '=IFERROR(vlookup(indirect("O" & row()),\'Lookup data\'!$A$2:$B$6,2,false))',
-          '=IF(ISBLANK(indirect("M" & row())), indirect("A" & row()), indirect("M" & row()))'
-        ]
+        libraries.push({name: name, owner: row.owner, version: row.current_version, source: row.source})
+        unless row.cvss.nil? || row.cvss == ""
+          vulns.push({library: name, identifier: row.cvss.split("[")[1].delete("]"), assigned_severity: row.cvss.split("[")[0].strip, url: row.dependabot_permalink})
+        end
+        new_versions.push({name: name, version: row.latest_version, major: row.major, minor: row.minor, patch: row.patch})
+        if row.dependency_graph.nil?
+          missing_dependency_keys.push(name)
+        else
+          dependencies.push(row.dependency_graph.deep_to_h)
+        end
       end
 
-      return data
-    end
-
-    def update_spreadsheet(spreadsheet_id, range_name, results)
-      service = Google::Apis::SheetsV4::SheetsService.new
-      service.authorization = ::Google::Auth::ServiceAccountCredentials.make_creds(scope: "https://www.googleapis.com/auth/spreadsheets")
+      binding.pry
 
-      clear_range = Google::Apis::SheetsV4::BatchClearValuesRequest.new
-      clear_range.ranges = [range_name]
-      service.batch_clear_values(spreadsheet_id, clear_range)
+      {
+        repository: repository,
+        libraries: libraries,
+        new_versions: new_versions,
+        vulnerabilities: vulns,
+        dependencies: dependencies,
+      }
+    end
 
-      value_range_object = Google::Apis::SheetsV4::ValueRange.new(range: range_name, values: results)
-      service.update_spreadsheet_value(spreadsheet_id, range_name, value_range_object, value_input_option: "USER_ENTERED")
+    def update_server(data)
+      uri = URI('http://127.0.0.1:4000/api/libraries/upload')
+      http = Net::HTTP.new(uri.host, uri.port)
+      req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json')
+      req["X-Upload-Key"] = ENV["UPLOAD_KEY"]
+      req.body = data.to_json
+      res = http.request(req)
+      puts "response #{res.body}"
     end
 
+    # def spreadsheet_data(results, source)
+    #   header_row = %w(name owner parent source current_version current_version_date latest_version latest_version_date major minor patch age cve note cve_label cve_severity note_lookup_key)
+    #   data = [header_row]
+    #
+    #   data << ["Updated: #{Time.now.utc}"]z
+    #
+    #   results.each do |name, row|
+    #     data << [
+    #       name,
+    #       row.owner,
+    #       row.parent,
+    #       source,
+    #       row.current_version,
+    #       row.current_version_date,
+    #       row.latest_version,
+    #       row.latest_version_date,
+    #       row.major,
+    #       row.minor,
+    #       row.patch,
+    #       row.age,
+    #       row.cvss,
+    #       '=IFERROR(concatenate(vlookup(indirect("Q" & row()),Notes!A:E,4,false), ":", concatenate(vlookup(indirect("Q" & row()),Notes!A:E,5,false))))',
+    #       '=IFERROR(vlookup(indirect("Q" & row()),Notes!A:E,4,false), IFERROR(trim(LEFT(INDIRECT("Q" & row()), SEARCH("[", INDIRECT("M" & row()))-1))))',
+    #       '=IFERROR(vlookup(indirect("O" & row()),\'Lookup data\'!$A$2:$B$6,2,false))',
+    #       '=IF(ISBLANK(indirect("M" & row())), indirect("A" & row()), indirect("M" & row()))'
+    #     ]
+    #   end
+    #
+    #   return data
+    # end
+
+    # def update_spreadsheet(spreadsheet_id, range_name, results)
+    #   service = Google::Apis::SheetsV4::SheetsService.new
+    #   service.authorization = ::Google::Auth::ServiceAccountCredentials.make_creds(scope: "https://www.googleapis.com/auth/spreadsheets")
+    #
+    #   clear_range = Google::Apis::SheetsV4::BatchClearValuesRequest.new
+    #   clear_range.ranges = [range_name]
+    #   service.batch_clear_values(spreadsheet_id, clear_range)
+    #
+    #   value_range_object = Google::Apis::SheetsV4::ValueRange.new(range: range_name, values: results)
+    #   service.update_spreadsheet_value(spreadsheet_id, range_name, value_range_object, value_input_option: "USER_ENTERED")
+    # end
+    # end
+
     def get_mode_summary(results, meta_data)
       mode_summary = ModeSummary.new
       mode_summary.one_major = 0
@@ -155,6 +221,8 @@ def get_mode_summary(results, meta_data)
       results.each do |hash_line|
         line = hash_line[1]
 
+        next if line.major.nil? # For libraries added for completeness of set, the following will all be empty
+
         if line.major.positive?
           mode_summary.one_major = mode_summary.one_major + 1 if line.major == 1
           mode_summary.two_major = mode_summary.two_major + 1 if line.major == 2

lib/library_version_analysis/github.rb

@@ -1,11 +1,21 @@
 require "graphql/client"
 require "graphql/client/http"
+require "pry-byebug"
 
 module LibraryVersionAnalysis
   class Github
     URL = "https://api.github.com/graphql".freeze
 
-    unless ENV['GITHUB_READ_API_TOKEN'].nil? || ENV['GITHUB_READ_API_TOKEN'].empty?
+    SOURCES = {
+      "NPM" => "npm",
+      "RUBYGEMS" => "gemfile",
+    }.freeze
+
+    if ENV['GITHUB_READ_API_TOKEN'].nil? || ENV['GITHUB_READ_API_TOKEN'].empty?
+      # THIS PREVENTS JOBBER FROM STARTING. REFACTOR SO NONE OF THIS IS DONE ON JOBBER STARTUP!!!!!
+      puts "ERROR: GITHUB_READ_API_TOKEN not set, aborting"
+      # exit(-1)
+    else
       HTTP_ADAPTER = GraphQL::Client::HTTP.new(URL) do
         def headers(_context)
           {
@@ -14,6 +24,7 @@ def headers(_context)
           }
         end
       end
+
       SCHEMA = GraphQL::Client.load_schema(HTTP_ADAPTER)
       CLIENT = GraphQL::Client.new(schema: SCHEMA, execute: HTTP_ADAPTER)
 
@@ -92,6 +103,7 @@ def get_dependabot_findings(parsed_results, meta_data, github_name, ecosystem)
             patch: 0,
             age: 0,
             cvss: cvss,
+            source: alert[:source]
           )
 
           parsed_results[package] = vv
@@ -133,7 +145,8 @@ def add_results(alerts, results, target_ecosystem)
             identifiers: alert.security_vulnerability.advisory.identifiers.map(&:value),
             severity: alert.security_vulnerability.severity,
             created_at: alert.created_at,
-            permalink: alert.security_vulnerability.advisory.permalink
+            permalink: alert.security_vulnerability.advisory.permalink,
+            source: SOURCES[ecosystem],
           }
         end
       end