diff --git a/.gitignore b/.gitignore index 8b18ace5..8b5efe4f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,4 @@ nosetests.xml # PyCharm project .idea/ -/*.egg_info +/*.egg-info/ diff --git a/.travis.yml b/.travis.yml index 61f978c0..d5196b43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - "2.6" - "2.7" install: - source configure etc/conf diff --git a/README.rst b/README.rst index f94bdba2..8ff8f8cc 100644 --- a/README.rst +++ b/README.rst @@ -71,14 +71,14 @@ And on Windows:: For instance on Linux the whole installation would be like this:: - $ wget https://github.com/dejacode/about-code-tool/archive/v2.0.1.zip - $ unzip v2.0.1.zip - $ cd about-code-tool-2.0.1/ + $ wget https://github.com/dejacode/about-code-tool/archive/v2.0.2.zip + $ unzip v2.0.2.zip + $ cd about-code-tool-2.0.2/ $ source configure On Windows, the whole installation would be like this: - * Download and extract https://github.com/dejacode/about-code-tool/archive/v2.0.1.zip + * Download and extract https://github.com/dejacode/about-code-tool/archive/v2.0.2.zip * open a command prompt and cd to the directory where the zip extraction directory * run configure diff --git a/about_code_tool/about.py b/about_code_tool/about.py index dbafc1b3..2a2bf064 100644 --- a/about_code_tool/about.py +++ b/about_code_tool/about.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -32,28 +32,31 @@ import csv from datetime import datetime from email.parser import HeaderParser -from os.path import basename, dirname, join, normpath, realpath import errno import httplib import logging +import ntpath import optparse import os +from os.path import basename, dirname, join, normpath, realpath import posixpath import socket import string import sys import urlparse -import ntpath -__version__ = '2.0.1' +on_windows = 'win32' in sys.platform +UNC_PREFIX = u'\\\\?\\' + +__version__ = '2.0.2' # See http://dejacode.org __about_spec_version__ = '1.0' __copyright__ = """ -Copyright (c) 2013-2014 nexB Inc. All rights reserved. +Copyright (c) 2013-2015 nexB Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -554,7 +557,14 @@ class AboutFile(object): """ def __init__(self, location=None): self.about_resource = None - self.location = location + # The os.path.abspath(None) will cause error in linux system. + # See https://bugs.python.org/issue22587 + # Note that the os.path.abspath is needed for windows when there + # is long path/filename. + if on_windows: + self.location = os.path.abspath(location) + else: + self.location = location self.parsed = None self.parsed_fields = None @@ -1082,7 +1092,10 @@ def get_dje_license_name(self): """ Return the about object's dje_license_name. """ - return self.parsed.get('dje_license_name', '') + try: + return self.parsed.get('dje_license_name', '') + except: + return '' def check_invalid_chars(field_name, line): """ @@ -1104,6 +1117,8 @@ def check_invalid_chars(field_name, line): warnings = Warn(IGNORED, field_name, line, msg) return invalid_chars, warnings +def posix_unc_prefix(): + return posix_path(u'\\\\?\\') class Collector(object): """ @@ -1144,6 +1159,23 @@ def collect(location): """ # FIXME: we should not accept both a file and dir location as input paths = [] + + if on_windows: + location = unicode(location) + """ + Convert a location to an absolute Window UNC path to support long paths + on Windows. Return the location unchanged if not on Windows. + See https://msdn.microsoft.com/en-us/library/aa365247.aspx + """ + if on_windows and not location.startswith(UNC_PREFIX): + location = UNC_PREFIX + os.path.abspath(location) + location = os.path.expanduser(location) + location = os.path.expandvars(location) + location = os.path.normpath(location) + location = os.path.abspath(location) + + assert os.path.exists(location) + if location: if os.path.isfile(location) and is_about_file(location): paths.append(location) @@ -1197,6 +1229,10 @@ def get_relative_path(self, location): """ user_loc = normpath(self.location) if os.path.isdir(self.normalized_location): + # Making sure both are in posix path format before + # doing any string partition. + location = posix_path(location) + user_loc = posix_path(user_loc) parent_name = basename(user_loc) subpath = '/' + parent_name + location.partition(user_loc)[2] if user_loc[-1] == '/': @@ -1241,6 +1277,10 @@ def get_about_context(self, about_object): if '\n' in about_object.get_dje_license_name(): msg = ('Multiple licenses is not supported. ' 'Skipping License generation.') + if on_windows: + if (about_object.location.startswith(posix_unc_prefix()) + or about_object.location.startswith(UNC_PREFIX)): + about_object.location = about_object.location.strip(posix_unc_prefix()).strip(UNC_PREFIX) err = Error(GENATTRIB, 'dje_license', about_object.location, msg) self.genattrib_errors.append(err) @@ -1258,6 +1298,10 @@ def get_about_context(self, about_object): and not '\n' in about_object.get_dje_license_name(): msg = ('No license_text found. ' 'Skipping License generation.') + if on_windows: + if (about_object.location.startswith(posix_unc_prefix()) + or about_object.location.startswith(UNC_PREFIX)): + about_object.location = about_object.location.strip(posix_unc_prefix()).strip(UNC_PREFIX) err = Error(GENATTRIB, 'license_text_file', about_object.location, msg) self.genattrib_errors.append(err) @@ -1314,6 +1358,9 @@ def generate_attribution(self, template_path=None, limit_to=None, verification=N break if not component_exist: + if on_windows: + if self.location.startswith(posix_unc_prefix()): + self.location = self.location.strip(posix_unc_prefix()) loc = self.location + component msg = ('The requested ABOUT file: %r does not exist. ' 'No attribution generated for this file.' % loc) @@ -1364,6 +1411,9 @@ def check_paths(self, paths): for path in paths: path = posix_path(path) afp = join(self.location, path) + if on_windows: + if afp.startswith(posix_unc_prefix()): + afp = afp.strip(posix_unc_prefix()) msg = ('The requested ABOUT file: %(afp)r does not exist. ' 'No attribution generated for this file.' % locals()) err = Error(GENATTRIB, 'about_file', path, msg) diff --git a/about_code_tool/genabout.py b/about_code_tool/genabout.py index 404f8c10..d9c38c90 100644 --- a/about_code_tool/genabout.py +++ b/about_code_tool/genabout.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - """ This is a tool to generate ABOUT files based on the input file. The input file should be a csv format which contains information about the @@ -22,29 +21,29 @@ from __future__ import print_function +from collections import namedtuple import copy import csv import errno import json import logging import optparse +from os import makedirs import os +from os.path import exists, dirname, join, abspath, isdir, normpath, basename, expanduser import shutil import sys import urllib import urllib2 - -from collections import namedtuple from urlparse import urljoin, urlparse -from os import makedirs -from os.path import exists, dirname, join, abspath, isdir, normpath, basename, expanduser import about -__version__ = '2.0.0' + +__version__ = '2.0.2' __copyright__ = """ -Copyright (c) 2013-2014 nexB Inc. All rights reserved. +Copyright (c) 2013-2015 nexB Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -135,7 +134,7 @@ def get_input_list(input_file): for row in csvfile: row_dict = {} for key in row: - row_dict[key.lower()] = row[key] + row_dict[key.lower()] = row[key].rstrip() input_list.append(row_dict) return input_list @@ -455,11 +454,12 @@ def pre_process_and_dje_license_dict(self, input_list, api_url, api_username, ap if not lic in license_dict: detail_list = [] detail = self.get_license_details_from_api(api_url, api_username, api_key, lic) - license_dict[lic] = detail[0] - line['dje_license_name'] = detail[0] dje_key = detail[1] + line['dje_license_key'] = dje_key + license_dict[dje_key] = detail[0] + line['dje_license_name'] = detail[0] license_context = detail [2] - line['dje_license_url'] = dje_lic_urn + lic + line['dje_license_url'] = dje_lic_urn + dje_key detail_list.append(dje_key) detail_list.append(license_context) key_text_dict[detail[0]] = detail_list @@ -515,7 +515,14 @@ def pre_generation(self, gen_location, input_list, action_num): about_file_location = join(gen_location, file_location) about_file_dir = dirname(about_file_location) if not os.path.exists(about_file_dir): - makedirs(about_file_dir) + # Check for invalid file path + try: + makedirs(about_file_dir) + except: + msg = 'Invalid ABOUT file path.' + self.errors.append(Error(VALUE, 'about_file_path', + about_file_dir, msg)) + continue about_file_exist = _exists(about_file_location) if about_file_exist: if action_num == ACTION_DO_NOTHING_IF_ABOUT_FILE_EXIST: @@ -604,6 +611,11 @@ def format_output(input_list): value = about_dict_list[item].replace('\n', '\n ') if (value or item in about.MANDATORY_FIELDS) and not item\ in about.ERROR_WARN_FIELDS and not item == 'about_resource': + # It will cause error if value has different coding + try: + value = unicode(value, errors='ignore') + except: + pass context += item + ': ' + value + '\n' component.append(about_file_location) @@ -614,6 +626,8 @@ def format_output(input_list): @staticmethod def write_output(output): for about_file_location, context in output: + if about.on_windows: + about_file_location = about.UNC_PREFIX + os.path.abspath(about_file_location) if _exists(about_file_location): os.remove(about_file_location) with open(about_file_location, 'wb') as output_file: @@ -855,6 +869,10 @@ def main(parser, options, args): sys.exit(errno.EINVAL) if gen_license: + # Strip the ' and " for api_url, api_username and api_key from input + api_url = api_url.strip("'").strip("\"") + api_username = api_username.strip("'").strip("\"") + api_key = api_key.strip("'").strip("\"") dje_license_dict = gen.pre_process_and_dje_license_dict(input_list, api_url, api_username, diff --git a/about_code_tool/genattrib.py b/about_code_tool/genattrib.py index 30544d19..3c95fe24 100644 --- a/about_code_tool/genattrib.py +++ b/about_code_tool/genattrib.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -43,12 +43,12 @@ logger.addHandler(handler) file_logger = logging.getLogger(__name__ + '_file') -__version__ = '2.0.0' +__version__ = '2.0.2' __about_spec_version__ = '1.0.0' # See http://dejacode.org __copyright__ = """ -Copyright (c) 2013-2014 nexB Inc. All rights reserved. +Copyright (c) 2013-2015 nexB Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/about_code_tool/tests/test_about.py b/about_code_tool/tests/test_about.py index b6cce448..0605e1f1 100644 --- a/about_code_tool/tests/test_about.py +++ b/about_code_tool/tests/test_about.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -27,9 +27,12 @@ from os.path import abspath, dirname, join, split from about_code_tool import about +from pip._vendor.distlib.wheel import to_posix +from about_code_tool.about import on_windows TESTDATA_DIR = join(abspath(dirname(__file__)), 'testdata') +UNC_PREFIX = u'\\\\?\\' def create_dir(location): @@ -134,21 +137,42 @@ def test_header_row_in_csv_output(self): def test_collect_can_collect_a_directory_tree(self): test_dir = 'about_code_tool/tests/testdata/DateTest' - expected = [('about_code_tool/tests/testdata/DateTest' - '/non-supported_date_format.ABOUT'), - ('about_code_tool/tests/testdata/DateTest' - '/supported_date_format.ABOUT')] + expected = [(os.path.abspath('about_code_tool/tests/testdata/DateTest' + '/non-supported_date_format.ABOUT')), + (os.path.abspath('about_code_tool/tests/testdata/DateTest' + '/supported_date_format.ABOUT'))] result = about.Collector.collect(test_dir) + if on_windows: + expected = [(to_posix(UNC_PREFIX + os.path.abspath('about_code_tool/tests/testdata/DateTest' + '/non-supported_date_format.ABOUT'))), + (to_posix(UNC_PREFIX + os.path.abspath('about_code_tool/tests/testdata/DateTest' + '/supported_date_format.ABOUT')))] self.assertEqual(sorted(expected), sorted(result)) def test_collect_can_collect_a_single_file(self): test_file = ('about_code_tool/tests/testdata/thirdparty' '/django_snippets_2413.ABOUT') - expected = ['about_code_tool/tests/testdata/thirdparty' - '/django_snippets_2413.ABOUT'] + expected = [os.path.abspath('about_code_tool/tests/testdata/thirdparty' + '/django_snippets_2413.ABOUT')] result = about.Collector.collect(test_file) + if on_windows: + expected = [to_posix(UNC_PREFIX + os.path.abspath('about_code_tool/tests/testdata/thirdparty' + '/django_snippets_2413.ABOUT'))] self.assertEqual(expected, result) + def test_collect_can_collect_a_long_directory_tree(self): + test_dir = 'about_code_tool/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/' + expected = [os.path.abspath(('about_code_tool/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' + '/non-supported_date_format.ABOUT'))] + result = about.Collector.collect(test_dir) + if on_windows: + # For some reasons, the os.path.abspath doesn't work if I have long + # path in the parameter. Therefore, I just append to long path + # after the os.path.abspath() + expected = [to_posix(UNC_PREFIX + os.path.abspath('about_code_tool') + '/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' + '/non-supported_date_format.ABOUT')] + self.assertEqual(sorted(expected), sorted(result)) + def test_collector_errors_encapsulation(self): test_file = 'about_code_tool/tests/testdata/DateTest' collector = about.Collector(test_file) diff --git a/about_code_tool/tests/test_genabout.py b/about_code_tool/tests/test_genabout.py index 03db877c..87abae55 100644 --- a/about_code_tool/tests/test_genabout.py +++ b/about_code_tool/tests/test_genabout.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -41,6 +41,16 @@ def test_get_input_list(self): result = gen.get_input_list(test_file) self.assertEqual(expected, result) + def test_get_input_with_trailing_spaces(self): + gen = genabout.GenAbout() + test_file = join(TESTDATA_DIR, 'test_files_for_genabout/about_with_trailling_spaces.csv') + expected = [{'about_file': 'about.c', + 'about_resource': '.', + 'name': 'ABOUT tool', + 'version': '0.8.1'}] + result = gen.get_input_list(test_file) + self.assertEqual(expected, result) + def test_get_input_list_covert_all_keys_to_lower(self): gen = genabout.GenAbout() test_input = join(TESTDATA_DIR, 'test_files_for_genabout' diff --git a/about_code_tool/tests/test_genattrib.py b/about_code_tool/tests/test_genattrib.py index 575e1101..701c2726 100644 --- a/about_code_tool/tests/test_genattrib.py +++ b/about_code_tool/tests/test_genattrib.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) 2013-2015 nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/about_code_tool/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/non-supported_date_format.ABOUT b/about_code_tool/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/non-supported_date_format.ABOUT new file mode 100644 index 00000000..11211ca2 --- /dev/null +++ b/about_code_tool/tests/testdata/longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/non-supported_date_format.ABOUT @@ -0,0 +1,4 @@ +name: distribute +version: 1.1 +about_resource: distribute_setup.py +date:01/08/2013 \ No newline at end of file diff --git a/about_code_tool/tests/testdata/test_files_for_genabout/about_with_trailling_spaces.csv b/about_code_tool/tests/testdata/test_files_for_genabout/about_with_trailling_spaces.csv new file mode 100644 index 00000000..d1672728 --- /dev/null +++ b/about_code_tool/tests/testdata/test_files_for_genabout/about_with_trailling_spaces.csv @@ -0,0 +1,2 @@ +about_file,about_resource,name,version +about.c ,.,ABOUT tool,0.8.1 diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 5a8ed1ef..8999392e 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -1,3 +1,19 @@ +2015-07-06 Chin-Yeung Li + + Release 2.0.2 + + * Handle input's encoding issues + * Better error handling + * Writing to and reading from Windows OS with paths > 255 chars + + +2015-06-08 Chin-Yeung Li + + Release 2.0.1 + + * Fixes the configure scripts and updates basic documentation. + + 2015-03-06 Chin-Yeung Li Release 2.0.0