Merge pull request #110 from goldmann/multiple-hashes

Add support for multiple hashing algorithms

dogen/generator.py

@@ -16,6 +16,8 @@
 from dogen import version, DEFAULT_SCRIPT_EXEC, DEFAULT_SCRIPT_USER
 from dogen.errors import Error
 
+SUPPORTED_HASH_ALGORITHMS = ['sha256', 'sha1', 'md5']
+
 class Generator(object):
     def __init__(self, log, args, plugins=[]):
         self.log = log
@@ -289,11 +291,29 @@ def handle_sources(self):
                 target = basename
 
             filename = ("%s/%s" % (self.output, target))
+
             passed = False
+            algorithms = []
+
+            md5sum = source.get('md5sum')
+
+            if md5sum:
+                self.log.warn("The 'md5sum' key is deprecated, please use 'md5' for %s. Or better switch to sha256 or sha1." % url)
+
+                # Backwards compatibility for md5sum
+                if not source.get('md5'):
+                    source['md5'] = md5sum
+
+            for supported_algorithm in SUPPORTED_HASH_ALGORITHMS:
+                if not source.get(supported_algorithm):
+                    continue
+
+                algorithms.append(supported_algorithm)
+
             try:
-                if os.path.exists(filename):
-                    if source.get('md5sum'):
-                        self.check_sum(filename, source['md5sum'])
+                if os.path.exists(filename) and algorithms:
+                    for algorithm in algorithms:
+                        self.check_sum(filename, source[algorithm], algorithm)
                     passed = True
             except Exception as e:
                 self.log.warn(str(e))
@@ -303,20 +323,36 @@ def handle_sources(self):
                 sources_cache = os.environ.get("DOGEN_SOURCES_CACHE")
 
                 if sources_cache:
-                    url = sources_cache.replace('#hash#', source['md5sum']).replace('#algorithm#', 'md5')
+                    url = sources_cache.replace('#filename#', basename)
+
+                    if algorithms:
+                        if len(algorithms) > 1:
+                            self.log.warn("You specified multiple algorithms for '%s' url, but only '%s' will be used to fetch it from cache" % (url, algorithms[0]))
+
+                        url = url.replace('#hash#', source[algorithms[0]]).replace('#algorithm#', algorithms[0])
+
                     self.log.info("Using '%s' as cached location for artifact" % url)
 
                 self._fetch_file(url, filename)
 
-                if source.get('md5sum'):
-                    self.check_sum(filename, source['md5sum'])
-                    self.cfg['artifacts'][target] = "md5:%s" % source['md5sum']
+                if algorithms:
+                    for algorithm in algorithms:
+                        self.check_sum(filename, source[algorithm], algorithm)
+                    self.cfg['artifacts'][target] = "%s:%s" % (algorithms[0], source[algorithms[0]])
                 else:
                     self.cfg['artifacts'][target] = None
 
-    def check_sum(self, filename, checksum):
-        self.log.info("Checking '%s' MD5 hash..." % os.path.basename(filename))
-        filesum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
+    def check_sum(self, filename, checksum, algorithm):
+        self.log.info("Checking '%s' %s hash..." % (os.path.basename(filename), algorithm))
+
+        hash = getattr(hashlib, algorithm)()
+
+        with open(filename, "rb") as f:
+            for chunk in iter(lambda: f.read(65536), b""):
+                hash.update(chunk)
+        filesum = hash.hexdigest()
+
         if filesum.lower() != checksum.lower():
-            raise Exception("The md5sum computed for the '%s' file ('%s') doesn't match the '%s' value" % (filename, filesum, checksum))
+            raise Exception("The %s computed for the '%s' file ('%s') doesn't match the '%s' value" % (algorithm, filename, filesum, checksum))
+
         self.log.debug("MD5 hash is correct.")

dogen/schema/kwalify_schema.yaml

@@ -70,6 +70,9 @@ map:
       - map:
           url: {type: str}
           md5sum: {type: str}
+          md5: {type: str}
+          sha1: {type: str}
+          sha256: {type: str}
           target: {type: str}
   packages:
     seq:

tests/schemas/good/openshift_amq_6.2_image.yaml

@@ -52,9 +52,9 @@ scripts:
       exec: install.sh
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/openshift-activemq-plugin-1.0.0.Final-redhat-1.jar
-      md5sum: 325fbbaff0f7dbea42203346d9c3bf98
+      md5: 325fbbaff0f7dbea42203346d9c3bf98
     - url: http://redacted/jboss-dmr-1.2.2.Final-redhat-1.jar
-      md5sum: 8df4cbf6f39c3bce21de16ad708084d5
+      md5: 8df4cbf6f39c3bce21de16ad708084d5
 

tests/schemas/good/openshift_datagrid_6.5_image.yaml

@@ -151,16 +151,16 @@ scripts:
       user: 185
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/javax.json-1.0.4.jar
-      md5sum: 569870f975deeeb6691fcb9bc02a9555
+      md5: 569870f975deeeb6691fcb9bc02a9555
     - url: http://redacted/jboss-logmanager-ext-1.0.0.Alpha2-redhat-1.jar
-      md5sum: 7c743e35463db5f55f415dd666d705c5
+      md5: 7c743e35463db5f55f415dd666d705c5
     - url: http://redacted/openshift-ping-common-1.0.0.Final-redhat-1.jar
-      md5sum: bafa4db7efe4082d76cde2fa9499bf84
+      md5: bafa4db7efe4082d76cde2fa9499bf84
     - url: http://redacted/openshift-ping-dns-1.0.0.Final-redhat-1.jar
-      md5sum: 71bbfdf795a2c65e4473df242f765490
+      md5: 71bbfdf795a2c65e4473df242f765490
     - url: http://redacted/openshift-ping-kube-1.0.0.Final-redhat-1.jar
-      md5sum: 145add030a89c3ed588dce27b0f24999
+      md5: 145add030a89c3ed588dce27b0f24999
     - url: http://redacted/oauth-20100527.jar
-      md5sum: 91c7c70579f95b7ddee95b2143a49b41
+      md5: 91c7c70579f95b7ddee95b2143a49b41

tests/schemas/good/openshift_eap_6.4_image.yaml

@@ -98,22 +98,22 @@ scripts:
       user: 185
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/javax.json-1.0.4.jar
-      md5sum: 569870f975deeeb6691fcb9bc02a9555
+      md5: 569870f975deeeb6691fcb9bc02a9555
     - url: http://redacted/jboss-logmanager-ext-1.0.0.Alpha2-redhat-1.jar
-      md5sum: 7c743e35463db5f55f415dd666d705c5
+      md5: 7c743e35463db5f55f415dd666d705c5
     - url: http://redacted/openshift-ping-common-1.0.0.Final-redhat-1.jar
-      md5sum: bafa4db7efe4082d76cde2fa9499bf84
+      md5: bafa4db7efe4082d76cde2fa9499bf84
     - url: http://redacted/openshift-ping-dns-1.0.0.Final-redhat-1.jar
-      md5sum: 71bbfdf795a2c65e4473df242f765490
+      md5: 71bbfdf795a2c65e4473df242f765490
     - url: http://redacted/openshift-ping-kube-1.0.0.Final-redhat-1.jar
-      md5sum: 145add030a89c3ed588dce27b0f24999
+      md5: 145add030a89c3ed588dce27b0f24999
     - url: http://redacted/oauth-20100527.jar
-      md5sum: 91c7c70579f95b7ddee95b2143a49b41
+      md5: 91c7c70579f95b7ddee95b2143a49b41
     - url: http://redacted/activemq-rar-5.11.0.redhat-621084.rar
-      md5sum: 207e17ac8102c93233fe2764d1fe8499
+      md5: 207e17ac8102c93233fe2764d1fe8499
     - url: http://redacted/rh-sso-7.0.0-eap6-adapter.zip
-      md5sum: 6fd81306ea4297307dcc5f51712e5f95
+      md5: 6fd81306ea4297307dcc5f51712e5f95
     - url: http://redacted/rh-sso-7.0.0-saml-eap6-adapter.zip
-      md5sum: 3b953c114dd09f86e71e18cd57d8af56
+      md5: 3b953c114dd09f86e71e18cd57d8af56

tests/schemas/good/openshift_eap_7.0_image.yaml

@@ -107,22 +107,22 @@ scripts:
       user: 185
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/javax.json-1.0.4.jar
-      md5sum: 569870f975deeeb6691fcb9bc02a9555
+      md5: 569870f975deeeb6691fcb9bc02a9555
     - url: http://redacted/jboss-logmanager-ext-1.0.0.Alpha2-redhat-1.jar
-      md5sum: 7c743e35463db5f55f415dd666d705c5
+      md5: 7c743e35463db5f55f415dd666d705c5
     - url: http://redacted/openshift-ping-common-1.0.0.Final-redhat-1.jar
-      md5sum: bafa4db7efe4082d76cde2fa9499bf84
+      md5: bafa4db7efe4082d76cde2fa9499bf84
     - url: http://redacted/openshift-ping-dns-1.0.0.Final-redhat-1.jar
-      md5sum: 71bbfdf795a2c65e4473df242f765490
+      md5: 71bbfdf795a2c65e4473df242f765490
     - url: http://redacted/openshift-ping-kube-1.0.0.Final-redhat-1.jar
-      md5sum: 145add030a89c3ed588dce27b0f24999
+      md5: 145add030a89c3ed588dce27b0f24999
     - url: http://redacted/oauth-20100527.jar
-      md5sum: 91c7c70579f95b7ddee95b2143a49b41
+      md5: 91c7c70579f95b7ddee95b2143a49b41
     - url: http://redacted/activemq-rar-5.11.0.redhat-621084.rar
-      md5sum: 207e17ac8102c93233fe2764d1fe8499
+      md5: 207e17ac8102c93233fe2764d1fe8499
     - url: http://redacted/rh-sso-7.0.0-eap7-adapter.zip
-      md5sum: 1542c1014d9ebc24522839a5fa8bee4d
+      md5: 1542c1014d9ebc24522839a5fa8bee4d
     - url: http://redacted/rh-sso-7.0.0-saml-eap7-adapter.zip
-      md5sum: ce858a47c707b362a968ffd5c66768dd
+      md5: ce858a47c707b362a968ffd5c66768dd

tests/schemas/good/openshift_fuse-camel_6.3_image.yaml

@@ -101,22 +101,22 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/javax.json-1.0.4.jar
-      md5sum: 569870f975deeeb6691fcb9bc02a9555
+      md5: 569870f975deeeb6691fcb9bc02a9555
     - url: http://redacted/jboss-logmanager-ext-1.0.0.Alpha2-redhat-1.jar
-      md5sum: 7c743e35463db5f55f415dd666d705c5
+      md5: 7c743e35463db5f55f415dd666d705c5
     - url: http://redacted/openshift-ping-common-1.0.0.Final-redhat-1.jar
-      md5sum: bafa4db7efe4082d76cde2fa9499bf84
+      md5: bafa4db7efe4082d76cde2fa9499bf84
     - url: http://redacted/openshift-ping-dns-1.0.0.Final-redhat-1.jar
-      md5sum: 71bbfdf795a2c65e4473df242f765490
+      md5: 71bbfdf795a2c65e4473df242f765490
     - url: http://redacted/openshift-ping-kube-1.0.0.Final-redhat-1.jar
-      md5sum: 145add030a89c3ed588dce27b0f24999
+      md5: 145add030a89c3ed588dce27b0f24999
     - url: http://redacted/oauth-20100527.jar
-      md5sum: 91c7c70579f95b7ddee95b2143a49b41
+      md5: 91c7c70579f95b7ddee95b2143a49b41
     - url: http://redacted/activemq-rar-5.11.0.redhat-621084.rar
-      md5sum: 207e17ac8102c93233fe2764d1fe8499
+      md5: 207e17ac8102c93233fe2764d1fe8499
     - url: http://redacted/rh-sso-7.0.0-eap6-adapter.zip
-      md5sum: 6fd81306ea4297307dcc5f51712e5f95
+      md5: 6fd81306ea4297307dcc5f51712e5f95
     - url: http://redacted/rh-sso-7.0.0-saml-eap6-adapter.zip
-      md5sum: 3b953c114dd09f86e71e18cd57d8af56
+      md5: 3b953c114dd09f86e71e18cd57d8af56

tests/schemas/good/openshift_kieserver_6.2_image.yaml

@@ -130,11 +130,11 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jboss-bpmsuite-6.2.1.GA-redhat-2-deployable-eap6.x.zip
-      md5sum: b63c7dfe82a44a140cce3a824c8c2e90
+      md5: b63c7dfe82a44a140cce3a824c8c2e90
     - url: http://redacted/openshift-kieserver-common-1.0.2.Final-redhat-1.jar
-      md5sum: 5858103206d0bcc4695aad38a7430c75
+      md5: 5858103206d0bcc4695aad38a7430c75
     - url: http://redacted/openshift-kieserver-jms-1.0.2.Final-redhat-1.jar
-      md5sum: 4a80b12399c49a1d274bbd1c62b49b65
+      md5: 4a80b12399c49a1d274bbd1c62b49b65
     - url: http://redacted/openshift-kieserver-web-1.0.2.Final-redhat-1.jar
-      md5sum: adf602d027020b30cc5743d3d5d8e2f7
+      md5: adf602d027020b30cc5743d3d5d8e2f7
 

tests/schemas/good/openshift_kieserver_6.3_image.yaml

@@ -131,12 +131,12 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jboss-bpmsuite-6.3.0.GA-deployable-eap6.x.zip
-      md5sum: 4e283717b0f295adf7025971065d6db8
+      md5: 4e283717b0f295adf7025971065d6db8
     - url: http://redacted/jboss-bpmsuite-6.3.0.GA-supplementary-tools.zip
-      md5sum: b3d135e2d297f1e89d9ff8357c1e9aac
+      md5: b3d135e2d297f1e89d9ff8357c1e9aac
     - url: http://redacted/openshift-kieserver-common-1.0.2.Final-redhat-1.jar
-      md5sum: 5858103206d0bcc4695aad38a7430c75
+      md5: 5858103206d0bcc4695aad38a7430c75
     - url: http://redacted/openshift-kieserver-jms-1.0.2.Final-redhat-1.jar
-      md5sum: 4a80b12399c49a1d274bbd1c62b49b65
+      md5: 4a80b12399c49a1d274bbd1c62b49b65
     - url: http://redacted/openshift-kieserver-web-1.0.2.Final-redhat-1.jar
-      md5sum: adf602d027020b30cc5743d3d5d8e2f7
+      md5: adf602d027020b30cc5743d3d5d8e2f7

tests/schemas/good/openshift_sso_7.0_image.yaml

@@ -97,17 +97,17 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/javax.json-1.0.4.jar
-      md5sum: 569870f975deeeb6691fcb9bc02a9555
+      md5: 569870f975deeeb6691fcb9bc02a9555
     - url: http://redacted/jboss-logmanager-ext-1.0.0.Alpha2-redhat-1.jar
-      md5sum: 7c743e35463db5f55f415dd666d705c5
+      md5: 7c743e35463db5f55f415dd666d705c5
     - url: http://redacted/openshift-ping-common-1.0.0.Final-redhat-1.jar
-      md5sum: bafa4db7efe4082d76cde2fa9499bf84
+      md5: bafa4db7efe4082d76cde2fa9499bf84
     - url: http://redacted/openshift-ping-dns-1.0.0.Final-redhat-1.jar
-      md5sum: 71bbfdf795a2c65e4473df242f765490
+      md5: 71bbfdf795a2c65e4473df242f765490
     - url: http://redacted/openshift-ping-kube-1.0.0.Final-redhat-1.jar
-      md5sum: 145add030a89c3ed588dce27b0f24999
+      md5: 145add030a89c3ed588dce27b0f24999
     - url: http://redacted/oauth-20100527.jar
-      md5sum: 91c7c70579f95b7ddee95b2143a49b41
+      md5: 91c7c70579f95b7ddee95b2143a49b41
 

tests/schemas/good/openshift_webserver-tomcat7_3.0_image.yaml

@@ -94,10 +94,10 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/json-smart-1.1.1.jar
-      md5sum: c382c9109020d001b96329c2057ba933
+      md5: c382c9109020d001b96329c2057ba933
     - url: http://redacted/commons-lang-2.6.0.redhat-4.jar
-      md5sum: 0da0fbfb0ff2160df3a4832d28003361
+      md5: 0da0fbfb0ff2160df3a4832d28003361
     - url: http://redacted/jsonevent-layout-1.7-redhat-1.jar
-      md5sum: 08f9aa037ac91c4aaa0d5dabf143a60e
+      md5: 08f9aa037ac91c4aaa0d5dabf143a60e

tests/schemas/good/openshift_webserver-tomcat8_3.0_image.yaml

@@ -94,10 +94,10 @@ scripts:
       exec: configure.sh
 sources:
     - url: http://redacted/jolokia-jvm-1.3.2.redhat-1-agent.jar
-      md5sum: 1b996b9083f537917b307309b0e2f16d
+      md5: 1b996b9083f537917b307309b0e2f16d
     - url: http://redacted/json-smart-1.1.1.jar
-      md5sum: c382c9109020d001b96329c2057ba933
+      md5: c382c9109020d001b96329c2057ba933
     - url: http://redacted/commons-lang-2.6.0.redhat-4.jar
-      md5sum: 0da0fbfb0ff2160df3a4832d28003361
+      md5: 0da0fbfb0ff2160df3a4832d28003361
     - url: http://redacted/jsonevent-layout-1.7-redhat-1.jar
-      md5sum: 08f9aa037ac91c4aaa0d5dabf143a60e
+      md5: 08f9aa037ac91c4aaa0d5dabf143a60e