Merge pull request #493 from kbrock/update-ansible-versions

Python requirements update

bin/requirements.rb

@@ -0,0 +1,207 @@
+#!/usr/bin/env ruby
+
+# This script takes the existing requirements.txt file
+# and updates it with the version for our supported packages
+#
+# USAGE:
+# 1. Setup environment
+#
+#     source /var/lib/manageiq/venv/bin/activate
+#     upload bin/requirements.rb and config/requirements.txt to /tmp
+#     chmod 755 requirements.rb
+#
+# 1. Get all module requirements (don't include documentation or testing ones)
+#
+#     ./requirements.rb ./requirements.txt /usr/lib/python3.9/site-packages/ansible_collections/ > new_requirements.txt
+#
+# 2. Resolve conflicts and determine if new one is correct
+#
+#     diff {,new_}requirements.txt
+#     # cp new_requirements.txt requirements.txt
+#
+
+class ParseRequirements
+  PACKAGES=%w[
+    amazon/aws/requirements.txt
+    ansible/netcommon/requirements.txt
+    ansible/utils/requirements.txt
+    awx/awx/requirements.txt
+    azure/azcollection/requirements-azure.txt
+    cisco/intersight/requirements.txt
+    community/aws/requirements.txt
+    community/okd/requirements.txt
+    community/vmware/requirements.txt
+    google/cloud/requirements.txt
+    kubernetes/core/requirements.txt
+    openstack/cloud/requirements.txt
+    ovirt/ovirt/requirements.txt
+    theforeman/foreman/requirements.txt
+    vmware/vmware_rest/requirements.txt
+  ]
+  attr_reader :filenames, :non_modules, :final
+
+  def initialize
+    @filenames = []
+    @non_modules = []
+
+    @final = {}
+  end
+
+  def add_target(filename)
+    if Dir.exist?(filename)
+      add_dir(filename)
+    elsif File.exist?(filename)
+      add_file(filename)
+    else
+      $stderr.puts("File not found: #{filename}")
+    end
+  end
+
+  def add_file(filename)
+    @filenames << filename
+    @non_modules << filename unless filename.include?("ansible_collections")
+
+    self
+  end
+
+  def add_dir(dirname)
+    dirname = dirname[0..-2] if dirname.end_with?("/")
+    PACKAGES.each do |package|
+      filename = "#{dirname}/#{package}"
+      if File.exist?(filename)
+        @filenames << filename
+      else
+        $stderr.puts("NOTICE: missing #{filename}")
+      end
+    end
+
+    self
+  end
+
+  def parse
+    filenames.each do |fn|
+      # the list of requirements-files can have items commented out - ignore those
+
+      mod = module_name_from_filename(fn)
+      IO.foreach(fn, chomp: true).each do |line|
+        lib, ver = parse_line(line)
+        next unless lib
+
+        # skip git libraries. git>= line from vsphere gave us problems
+        next if lib.start_with?("git")
+
+        # Do not version packages that are provided by the system: requests, requests[security], pyspnego[kerberos]
+        ver = "" if lib.match?(/^requests($|\[)/) || lib.start_with?("pyspnego")
+
+        final[lib] ||= {}
+        (final[lib][ver] ||= []) << mod
+      end
+    end
+
+    self
+  end
+
+  def output
+    result = final.flat_map do |lib, vers|
+      # consolidate multiple versioning rules
+      if vers.size > 1
+        max_key, *all_keys = vers.keys
+        all_keys.each do |alt|
+          higher, lower, conflict = version_compare(alt, max_key)
+          # There is a conflict when we have conflicting requirements. eg: >=2.0 and ==1.0
+          # We are displaying all comparisons/winners to verify the comparison algorithm works (skipping when merging a blank - no change of errors there)
+          $stderr.puts "#{lib}: #{higher} > #{lower} #{"CONFLICT" if conflict}" if lower != ""
+          vers[higher].concat(vers.delete(lower))
+          max_key = higher
+        end
+      end
+
+      vers.map do |(ver, modules)|
+        # if we pass in a previous requirements.txt, lets not mention it
+        # exception: display if it is only mentioned in a previous requirements.txt file
+        modules.delete("legacy") if modules.size > 1
+        "#{lib}#{ver} # #{modules.join(", ")}"
+      end
+    end.sort.join("\n")
+
+    puts result
+  end
+
+  private
+
+  def module_name_from_filename(fn)
+    if non_modules.include?(fn)
+      "legacy"
+    else
+      fn.gsub(%r{.*ansible_collections/}, "")
+        .gsub(%r{/requirements.*}, "")
+    end
+  end
+
+  def parse_line(line)
+    line.downcase!
+    # TODO: do we want to keep legacy comments? Only useful for our requirements.txt file
+    line.gsub!(/#.*/, "")
+    line.strip!
+    return if line.empty?
+
+    # Some libraries list "python" instead of "python_version"
+    # Dropping since just listing the python version isn't useful
+    return if line.match?(/^python([ <=>]|_version)/)
+
+    # Some libraries list version "5+" instead of ">=5"
+    line.gsub!(/\([0-9.]*\)\+/, '>=\1')
+    line.gsub!("= ", "=")
+    # Ignore package requirements for older version of pythons (assumption here)
+    return if line.match?(/python_version ?[=<]/)
+
+    lib, ver = split_lib_ver(line)
+
+    # Note: Already normalized for lowercase
+    # Normalize library name with dash. All these characters are treated the same.
+    lib.gsub!(/[-_.]+/, "-")
+    ver ||= ""
+
+   # TODO: split off ;python_version in split_lib_version - evaluate it properly
+   return if ver.match?(/python_version *[=<]/)
+
+    [lib, ver]
+  end
+
+  # ipaddress>=1.0,<=2.0;python_version<3.0
+  # currently returning  "ipaddress", ">=1.0,<=2.0;python_version<3.0"
+  # @return lib, version
+  def split_lib_ver(line)
+    # split on first space (or =)
+    # version can have multiple spaces
+    lib, ver = line.match(/([^ >=]*) ?(.*)/).captures
+
+    [lib, ver]
+  end
+
+  # @return [Numeric, Numeric, Boolean]
+  #   highest, lowest for version comparison
+  #   boolean is true if there is a conflict with the versions
+  def version_compare(a, b)
+    winner = a if a.start_with?("==")
+    winner = b if b.start_with?("==")
+    # due to the way zip works, we need the longer to be on the left of the split
+    a, b = b, a if a.split(".").length < b.split(".").length
+
+    # when comparing, drop off the >= or == stuff, just look at the numbers
+    # kinda assuming that we are dealing mostly with >=
+    # reminder <=> returns -1, 0, +1 like standard `cmp` functionality from c.
+    cmp = a.gsub(/^[=<>]+/, "").split(".").zip(b.gsub(/^[=<>]+/, "").split(".")).inject(0) { |acc, (v1, v2)| acc == 0 ? v1.to_i<=>v2.to_i : acc }
+
+    # ensure a >= b
+    a, b = b, a if cmp < 0
+
+    [a, b, winner && winner != a]
+  end
+end
+
+# {"lib" => {ver => [module]}}
+
+pr = ParseRequirements.new
+ARGV.each { |arg| pr.add_target(arg) }
+pr.parse.output

config/requirements.txt

@@ -1,51 +1,87 @@
 aiohttp # vmware/vmware_rest
-apache-libcloud==2.5.0
-asn1crypto==0.24.0
-azure-cli-core==2.0.35
-azure-graphrbac==0.40.0
-azure-keyvault==1.0.0
-azure-mgmt-authorization==0.51.1
-azure-mgmt-batch==5.0.1
-azure-mgmt-cdn==3.0.0
-azure-mgmt-compute==4.4.0
-azure-mgmt-containerinstance==1.4.0
-azure-mgmt-containerregistry==2.0.0
-azure-mgmt-containerservice==4.4.0
-azure-mgmt-cosmosdb==0.5.2
-azure-mgmt-devtestlabs==3.0.0
-azure-mgmt-dns==2.1.0
-azure-mgmt-hdinsight==0.1.0
-azure-mgmt-keyvault==1.1.0
-azure-mgmt-loganalytics==0.2.0
-azure-mgmt-marketplaceordering==0.1.0
-azure-mgmt-monitor==0.5.2
-azure-mgmt-network==2.3.0
-azure-mgmt-nspkg==2.0.0
-azure-mgmt-rdbms==1.4.1
-azure-mgmt-redis==5.0.0
-azure-mgmt-resource==2.1.0
-azure-mgmt-servicebus==0.5.3
-azure-mgmt-sql==0.10.0
-azure-mgmt-storage==3.1.0
-azure-mgmt-trafficmanager==0.50.0
-azure-mgmt-web==0.41.0
-azure-storage==0.35.1
-backports.ssl-match-hostname==3.5.0.1
+ansible-pylibssh>=0.2.0 # ansible/netcommon
+apache-libcloud # legacy
+awxkit # awx/awx
+azure-cli-core==2.34.0 # azure/azcollection
+azure-common==1.1.11 # azure/azcollection
+azure-containerregistry==1.1.0 # azure/azcollection
+azure-graphrbac==0.61.1 # azure/azcollection
+azure-identity==1.7.0 # azure/azcollection
+azure-keyvault==1.1.0 # azure/azcollection
+azure-mgmt-apimanagement==3.0.0 # azure/azcollection
+azure-mgmt-authorization==2.0.0 # azure/azcollection
+azure-mgmt-automation==1.0.0 # azure/azcollection
+azure-mgmt-batch==5.0.1 # azure/azcollection
+azure-mgmt-cdn==11.0.0 # azure/azcollection
+azure-mgmt-compute==26.1.0 # azure/azcollection
+azure-mgmt-containerinstance==9.0.0 # azure/azcollection
+azure-mgmt-containerregistry==9.1.0 # azure/azcollection
+azure-mgmt-containerservice==20.0.0 # azure/azcollection
+azure-mgmt-core==1.3.0 # azure/azcollection
+azure-mgmt-cosmosdb==6.4.0 # azure/azcollection
+azure-mgmt-datafactory==2.0.0 # azure/azcollection
+azure-mgmt-datalake-store==1.0.0 # azure/azcollection
+azure-mgmt-devtestlabs==9.0.0 # azure/azcollection
+azure-mgmt-dns==8.0.0 # azure/azcollection
+azure-mgmt-eventhub==10.1.0 # azure/azcollection
+azure-mgmt-hdinsight==9.0.0 # azure/azcollection
+azure-mgmt-iothub==2.2.0 # azure/azcollection
+azure-mgmt-keyvault==10.0.0 # azure/azcollection
+azure-mgmt-loganalytics==12.0.0 # azure/azcollection
+azure-mgmt-managedservices==6.0.0 # azure/azcollection
+azure-mgmt-managementgroups==1.0.0 # azure/azcollection
+azure-mgmt-marketplaceordering==1.1.0 # azure/azcollection
+azure-mgmt-monitor==3.0.0 # azure/azcollection
+azure-mgmt-network==19.1.0 # azure/azcollection
+azure-mgmt-notificationhubs==7.0.0 # azure/azcollection
+azure-mgmt-nspkg==2.0.0 # azure/azcollection
+azure-mgmt-privatedns==1.0.0 # azure/azcollection
+azure-mgmt-rdbms==10.0.0 # azure/azcollection
+azure-mgmt-recoveryservices==2.0.0 # azure/azcollection
+azure-mgmt-recoveryservicesbackup==3.0.0 # azure/azcollection
+azure-mgmt-redis==13.0.0 # azure/azcollection
+azure-mgmt-resource==21.1.0 # azure/azcollection
+azure-mgmt-search==8.0.0 # azure/azcollection
+azure-mgmt-servicebus==7.1.0 # azure/azcollection
+azure-mgmt-sql==3.0.1 # azure/azcollection
+azure-mgmt-storage==19.0.0 # azure/azcollection
+azure-mgmt-trafficmanager==1.0.0b1 # azure/azcollection
+azure-mgmt-web==6.1.0 # azure/azcollection
+azure-nspkg==2.0.0 # azure/azcollection
+azure-storage-blob==12.11.0 # azure/azcollection
 boto3>=1.18.0 # amazon/aws, community/aws
 botocore>=1.21.0 # amazon/aws, community/aws
-deprecation>=2.0
-google-auth==1.6.2
-ipaddress==1.0.23
-monotonic==1.4
-ncclient>=0.6.3
-netaddr==0.7.19
-openstacksdk==0.23.0
-ovirt-engine-sdk-python==4.2.4
-pexpect==4.6.0
-psutil==5.6.6 # Match the version installed directly into the venv
-pykerberos==1.2.1
+cryptography>=36.0.0 # cisco/intersight
+deprecation # legacy
+google-auth==1.6.2 # google/cloud
+google-cloud-storage # google/cloud
+grpcio # ansible/netcommon
+jsonpatch # kubernetes/core
+jsonschema # ansible/utils
+jxmlease # ansible/netcommon
+kubernetes>=12.0.0 # community/okd, kubernetes/core
+msrest==0.7.1 # azure/azcollection
+msrestazure==0.6.4 # azure/azcollection
+ncclient>=0.6.3 # ansible/netcommon
+netaddr==0.7.19 # ansible/netcommon, ansible/utils
+openstacksdk>=0.36,<0.99.0 # openstack/cloud
+ovirt-engine-sdk-python>=4.5.0 # ovirt/ovirt
+ovirt-imageio # ovirt/ovirt
+packaging # azure/azcollection
+paramiko==2.8.1 # ansible/netcommon
+pexpect # legacy
+protobuf # ansible/netcommon
+psutil # legacy
+python-dateutil>=2.7.0 # awx/awx
+pytz # awx/awx
 pyvmomi>=6.7.1 # community/vmware
-pywinrm # general requirement
-requests==2.25.1
-requests-credssp==0.1.0
-requests-kerberos==0.14.0
+pywinrm # legacy
+pyyaml # theforeman/foreman
+requests # google/cloud, theforeman/foreman
+requests-credssp # legacy
+requests-kerberos # legacy
+requests-oauthlib # community/okd, kubernetes/core
+requests[security] # azure/azcollection
+textfsm # ansible/utils
+ttp # ansible/utils
+xmltodict # ansible/netcommon, ansible/utils, azure/azcollection