From 7626c33068be82f8bede3e6781079130d9bae907 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 16 Jan 2024 12:00:00 +0100 Subject: [PATCH] To-be-split-up: Add pytype to CI to find Py3 bugs and fix issues to let pytype finish Signed-off-by: Bernhard Kaindl --- .github/workflows/main.yml | 2 + .pre-commit-config.yaml | 17 +- .pylintrc | 1 + Makefile | 4 + ocaml/message-switch/python/message_switch.py | 2 +- pyproject.toml | 106 +++ pytest.ini | 15 + run_pytype.py | 179 ++++ scripts/__init__.py | 0 scripts/backup-sr-metadata.py | 20 +- scripts/examples/python/XenAPI/XenAPI.py | 8 +- scripts/examples/python/exportimport.py | 6 +- scripts/examples/python/mini-xenrt.py | 2 +- .../python/monitor-unwanted-domains.py | 85 +- scripts/examples/python/provision.py | 4 +- scripts/examples/python/shell.py | 4 +- scripts/examples/smapiv2.py | 2 +- scripts/hatests | 34 +- scripts/hfx_filename | 11 +- scripts/mail-alarm | 16 +- scripts/nbd_client_manager.py | 6 +- scripts/perfmon | 8 +- scripts/plugins/extauth-hook-AD.py | 1 + scripts/restore-sr-metadata.py | 39 +- scripts/static-vdis | 3 +- scripts/static-vdisc | Bin 0 -> 15505 bytes scripts/test_mail-alarm.py | 11 +- scripts/test_static_vdis.py | 56 +- scripts/test_usb_scan.py | 4 +- scripts/unit_tests/__init__.py | 0 scripts/unit_tests/conftest.py | 5 +- scripts/unit_tests/import_helper.py | 74 ++ scripts/unit_tests/test_usb_reset.py | 9 +- scripts/usb_reset.py | 16 +- scripts/usb_scan.py | 13 +- scripts/yum/__init__.py | 0 scripts/yum/plugins.py | 781 ++++++++++++++++++ xcp/__init__.py | 25 + xcp/cmd.py | 135 +++ xcp/compat.py | 59 ++ xcp/logger.py | 150 ++++ xcp/logger.pyi | 15 + 42 files changed, 1751 insertions(+), 177 deletions(-) create mode 100644 pyproject.toml create mode 100644 pytest.ini create mode 100755 run_pytype.py create mode 100644 scripts/__init__.py create mode 100644 scripts/static-vdisc create mode 100644 scripts/unit_tests/__init__.py create mode 100644 scripts/unit_tests/import_helper.py create mode 100644 scripts/yum/__init__.py create mode 100755 scripts/yum/plugins.py create mode 100644 xcp/__init__.py create mode 100644 xcp/cmd.py create mode 100644 xcp/compat.py create mode 100644 xcp/logger.py create mode 100644 xcp/logger.pyi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee02c2d5c75..1004bb616f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,6 +73,8 @@ jobs: python-version: '3.10' cache: 'pip' + - run: sudo apt-get install -y libcurl4-nss-dev + - uses: actions/cache@v3 name: Setup cache for running pre-commit fast with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e283a4d3764..db50deea1dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,5 +57,18 @@ repos: rev: v1.8.0 hooks: - id: mypy - files: ^scripts/unit_tests/ - additional_dependencies: [pytest-mock, types-mock] \ No newline at end of file + files: ^scripts/(unit_tests|usb_reset.py) + additional_dependencies: [pytest-mock, types-mock] + + +- repo: local + hooks: + - id: pytype + name: pytype + entry: python3 run_pytype.py + types: [python] + verbose: true + language: python + pass_filenames: false + require_serial: true + additional_dependencies: [pandas, pytype, pyudev, XenAPI, urlgrabber] \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 0dbd24cd917..e6285148543 100644 --- a/.pylintrc +++ b/.pylintrc @@ -41,6 +41,7 @@ verbose=yes disable= bad-option-value, # old pylint for py2: ignore newer (unknown) pylint options bad-continuation, # old pylint warns about some modern black formatting + consider-using-f-string, useless-option-value, # new pylint has abaondoned these old options [BASIC] diff --git a/Makefile b/Makefile index 4d155619a57..800b57ead65 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,10 @@ ifneq ($(PY_TEST), NO) dune build @runtest-python --profile=$(PROFILE) endif +pytype: + pip3 install --upgrade pytype + ./run_pytype.py + stresstest: dune build @stresstest --profile=$(PROFILE) --no-buffer -j $(JOBS) diff --git a/ocaml/message-switch/python/message_switch.py b/ocaml/message-switch/python/message_switch.py index 460d4ee2e04..b3d870895c3 100755 --- a/ocaml/message-switch/python/message_switch.py +++ b/ocaml/message-switch/python/message_switch.py @@ -273,7 +273,7 @@ def get_reply(self, correlation_id): def set_listen_callback(self, listen_callback): self.listen_callback = listen_callback def run(self): - ack_to = -1L + ack_to = -1 timeout = 5.0 while True: messages = transfer(self.sock, self.reader, ack_to, timeout) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..f107cddb5e6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,106 @@ +# https://packaging.python.org/en/latest/specifications/pyproject-toml/ +[project] +name = "xen-api" +requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +license = {file = "LICENSE"} +keywords = ["xen-project", "Xen", "hypervisor", "libraries"] +maintainers = [ + {name = "Christian Lindig"}, + {name = "Edwin Török"}, + {name = "Rob Hoes"}, + {name = "Pau Ruiz Safont"}, +] +readme = "README.markdown" +# https://pypi.org/classifiers/ +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Operating System :: POSIX :: Linux :: XenServer Dom0", + "Operating System :: POSIX :: Linux :: XCP-ng Dom0", + "Programming Language :: ML", + "Programming Language :: Python :: Implementation :: CPython", +] + +[project.urls] +homepage = "https://github.com/xapi-project/xen-api" +repository = "https://github.com/xapi-project/xen-api" + +[tool.black] +line-length = 88 + +[tool.isort] +line_length = 88 +py_version = 27 +profile = "black" +combine_as_imports = true +ensure_newline_before_comments = false + +[tool.mypy] +# Note mypy has no config setting for PYTHONPATH, so you need to call it with: +# PYTHONPATH="scripts/examples/python:.:scripts:scripts/plugins:scripts/examples" +files = [ + "scripts/usb_reset.py", + "scripts/unit_tests", +] +pretty = true +error_summary = true +strict_equality = true +show_error_codes = true +show_error_context = true +# Check the contents of untyped functions in all modules by default: +check_untyped_defs = true +scripts_are_modules = true +python_version = "3.11" +warn_return_any = true +warn_unreachable = true +warn_unused_configs = true +warn_redundant_casts = true +disallow_any_explicit = false +disallow_any_generics = true +disallow_any_unimported = true +disallow_subclassing_any = true + +[tool.pytype] +inputs = [ + 'scripts/*.py', + 'scripts/', + "scripts/10resetvdis", + "scripts/Makefile", + "scripts/generate-iscsi-iqn", + "scripts/hatests", + "scripts/hfx_filename", + "scripts/host-display", + "scripts/mail-alarm", + "scripts/print-custom-templates", + "scripts/probe-device-for-file", + "scripts/xe-reset-networking", + "scripts/xe-scsi-dev-map", + 'scripts/examples/python', + # Don't add, it can't do "from .XenAPI import *" afterwards: + # 'scripts/examples/python/XenAPI', + # Not yet ported: + # "ocaml/message-switch/python", + # "ocaml/idl/ocaml_backend/python", + # "ocaml/xapi-storage/python", +] +xfail = [ + "scripts/perfmon", + "scripts/static-vdis", + "scripts/usb_scan.py", + "scripts/yum", + "scripts/yum/plugins.py", + "scripts/examples/python/monitor-unwanted-domains.py" +] +disable = [ + 'import-error', + "pyi-error", + "ignored-abstractmethod" +] +keepgoing = true +platform = "linux" +python_version = "3.10" +pythonpath = "scripts/examples/python:.:scripts:scripts/plugins:scripts/examples" +# pythonpath = "scripts/examples/python/XenAPI" +# ":scripts/examples/python:scripts/examples:scripts:." +# disable = ["ignored-type-comment"] +# overriding_parameter_count_checks = true +# use_enum_overlay = true \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..d8fe40b022b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,15 @@ +[pytest] + +# By default, show reports for failed tests: +addopts=-rF + +# Enable log display during test run (also known as “live logging”). +log_cli=True + +# Sets the minimum log message level that should be captured for live logging. +# The integer value or the names of the levels can be used. +# Lower it for debugging: +log_cli_level=FATAL + +# When on path is passwd, run the tests below the scripts directory: +testpaths=scripts/ diff --git a/run_pytype.py b/run_pytype.py new file mode 100755 index 00000000000..6608a0b34f9 --- /dev/null +++ b/run_pytype.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +import os +import re +import selectors +import shlex +import sys +from logging import INFO, basicConfig, info +from subprocess import PIPE, Popen +from typing import Dict, List, TextIO, Tuple + +import pandas as pd # type: ignore[import] +from toml import load + + +def generate_github_annotation(match: re.Match[str], branch_url: str) -> Tuple[str, Dict[str, str]]: + lineno = match.group(2) + code = match.group(5) + func = match.group(3) + msg = match.group(4) + assert isinstance(msg, str) + msg_splitpos = msg.find(" ", 21) + file = match.group(1) + linktext = os.path.basename(file).split(".")[0] + source_link = f"[`{linktext}`]({branch_url}/{file}#L{lineno})" + row = { + "Location": source_link, + "Function": f"`{func}`", + "Error code": code, + "Error message": msg[:msg_splitpos] + "
" + msg[msg_splitpos + 1 :], + "Error description": "", + } + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message + return f"::error file={file},line={lineno},title=pytype: {code}::{msg}", row + + +def filter_line(line, row): + if line.startswith("For more details, see"): + row["Error code"] = f"[{row['Error code']}]({line[22:]})" + return " " + line[22:] + if not row["Error description"]: + row["Error description"] = line.lstrip() + else: + row["Error description"] += " " + line.lstrip() + return ", " + line + + +def skip_uninteresting_lines(line: str) -> bool: + if not line or line[0] == "/" or line.startswith("FAILED:"): + return True + if line[0] == "[": + pos = line.rfind(os.getcwd()) + printfrom = pos + len(os.getcwd()) + 1 if pos > 0 else line.index("]") + 2 + info("PROGRESS: " + line[1:].split("]")[0] + ": " + line[printfrom:]) + return True + if line.startswith("ninja: "): + line = line[7:] + return bool( + ( + line.startswith("Entering") + or line.startswith("Leaving") + or line.startswith("Computing") + or line.startswith("Analyzing") + ) + ) + + +def run_pytype(command: List[str], branch_url: str, errorlog: TextIO, results): + info(" ".join(shlex.quote(arg) for arg in command)) + # When run in tox, pytype dumps debug messages to stderr. Point stderr to /dev/null: + popen = Popen(command, stdout=PIPE, stderr=PIPE, universal_newlines=True) + assert popen.stdout and popen.stderr + error = "" + row = {} # type: dict[str, str] + sel = selectors.DefaultSelector() + sel.register(popen.stdout, selectors.EVENT_READ) + sel.register(popen.stderr, selectors.EVENT_READ) + ok = True + while ok: + for key, _ in sel.select(): + line = key.fileobj.readline() # type: ignore + if not line: + ok = False + break + if key.fileobj is popen.stderr: + print(f"pytype: {line}", end="", file=sys.stderr) + continue + line = line.rstrip() + if skip_uninteresting_lines(line): + continue + info(line) + if row: + if line == "" or line[0] == " " or line.startswith("For more details, see"): + if line: + error += filter_line(line, row) + continue + errorlog.write( + error + + " (you should find an entry in the pytype results with links below)\n" + ) + results.append(row) + row = {} + error = "" + match = re.match( + r'File ".*libs/([^"]+)", line (\S+), in ([^:]+): (.*) \[(\S+)\]', line + ) + if match: + error, row = generate_github_annotation(match, branch_url) + if popen.stdout: + popen.stdout.close() + popen.wait() + return popen.returncode, results + + +def run_pytype_and_parse_annotations(xfail_files: List[str], branch_url: str): + """Send pytype errors to stdout and return results as pandas table + + Args: + xfail_files (List[str]): list of files to exclude from pytype checks + branch_url (str): Base URL of the git branch for file links in github annotations + """ + base_command = [ + "pytype", + "-j", + "auto", + ] + if xfail_files: + exclude_command = ["--exclude", " ".join(xfail_files)] + else: + exclude_command = [] + + err_code, results = run_pytype(base_command + exclude_command, branch_url, sys.stderr, []) + if err_code or len(results): + return err_code if err_code > 0 else len(results), results + for xfail_file in xfail_files: + err_code, results = run_pytype(base_command + [xfail_file], branch_url, sys.stdout, results) + if err_code == 0: + print("No errors in", xfail_file) + return err_code or len(results), results + +def to_markdown(me, fp, returncode, results, branch_url): + mylink = f"[{me}]({branch_url}/{me}.py)" + pytype_link = "[pytype](https://google.github.io/pytype)" + if len(results) or returncode: + fp.write(f"\n#### {mylink} reports these {pytype_link} error messages:\n") + fp.write(pd.DataFrame(results).to_markdown()) + else: + fp.write(f"\n#### Congratulations, {mylink} reports no {pytype_link} errors.\n") + fp.write("\n") + + +def setup_and_run_pytype_action(script_name: str): + config = load("pyproject.toml") + pytype = config["tool"].get("pytype") + xfail_files = pytype.get("xfail", []) if pytype else [] + repository_url = config["project"]["urls"]["repository"].strip(" /") + filelink_baseurl = repository_url + "/blob/master" + + # When running as a GitHub action, we want to use URL of the fork with the GitHub action: + server_url = os.environ.get("GITHUB_SERVER_URL", None) + repository = os.environ.get("GITHUB_REPOSITORY", None) + if server_url and repository: + # https://github.com/orgs/community/discussions/5251 only set on Pull requests: + branch = os.environ.get("GITHUB_HEAD_REF", None) or os.environ.get("GITHUB_REF_NAME", None) + filelink_baseurl = f"{server_url}/{repository}/blob/{branch}" + ret_code, results = run_pytype_and_parse_annotations(xfail_files, filelink_baseurl) + + # Write the panda table to a markdown output file: + summary_file = os.environ.get("GITHUB_STEP_SUMMARY", None) + if summary_file: + with open(summary_file, "w", encoding="utf-8") as fp: + to_markdown(script_name, fp, ret_code, results, filelink_baseurl) + else: + to_markdown(script_name, sys.stdout, ret_code, results, filelink_baseurl) + + +if __name__ == "__main__": + script_basename = os.path.basename(__file__).split(".")[0] + basicConfig(format=script_basename + ": %(message)s", level=INFO) + sys.exit(setup_and_run_pytype_action(script_name = script_basename)) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/backup-sr-metadata.py b/scripts/backup-sr-metadata.py index 2464d5c8761..1f3f033bb86 100644 --- a/scripts/backup-sr-metadata.py +++ b/scripts/backup-sr-metadata.py @@ -7,21 +7,15 @@ import sys import getopt import codecs -from xml.dom.minidom import Document +from xml.dom.minidom import Document # pytype: disable=pyi-error -def logout(): - try: - session.xenapi.session.logout() - except: - pass -atexit.register(logout) def usage(): - print >> sys.stderr, "%s [-f ]" % sys.argv[0] + print("%s [-f ]" % sys.argv[0], file=sys.stderr) sys.exit(1) def set_if_exists(xml, record, key): - if record.has_key(key): + if key in record: xml.setAttribute(key, record[key]) else: xml.setAttribute(key, "") @@ -32,8 +26,8 @@ def main(argv): try: opts, args = getopt.getopt(argv, "hf:", []) - except getopt.GetoptError, err: - print str(err) + except getopt.GetoptError as err: + print(str(err)) usage() outfile = None @@ -54,7 +48,7 @@ def main(argv): metaxml = doc.createElement("meta") doc.appendChild(metaxml) - for srref in srs.keys(): + for srref in list(srs.keys()): srrec = srs[srref] srxml = doc.createElement("sr") set_if_exists(srxml, srrec, 'uuid') @@ -70,7 +64,7 @@ def main(argv): set_if_exists(vdixml, vdirec, 'name_description') srxml.appendChild(vdixml) except: - print >> sys.stderr, "Failed to get VDI record for: %s" % vdiref + print("Failed to get VDI record for: %s" % vdiref, file=sys.stderr) metaxml.appendChild(srxml) diff --git a/scripts/examples/python/XenAPI/XenAPI.py b/scripts/examples/python/XenAPI/XenAPI.py index ab868e92188..a44a7b932bf 100644 --- a/scripts/examples/python/XenAPI/XenAPI.py +++ b/scripts/examples/python/XenAPI/XenAPI.py @@ -121,7 +121,9 @@ class Session(xmlrpclib.ServerProxy): def __init__(self, uri, transport=None, encoding=None, verbose=0, allow_none=1, ignore_ssl=False): - + if sys.version_info >= (3,): + verbose = verbose != 0 + allow_none = allow_none != 0 # Fix for CA-172901 (+ Python 2.4 compatibility) # Fix for context=ctx ( < Python 2.7.9 compatibility) if not (sys.version_info[0] <= 2 and sys.version_info[1] <= 7 and sys.version_info[2] <= 9 ) \ @@ -176,13 +178,15 @@ def _login(self, method, params): self.last_login_params = params self.API_version = self._get_api_version() except socket.error as e: - if e.errno == socket.errno.ETIMEDOUT: + if e.errno == socket.errno.ETIMEDOUT: # pytype: disable=module-attr raise xmlrpclib.Fault(504, 'The connection timed out') else: raise e def _logout(self): try: + if sys.version_info >= (3,): + raise NotImplementedError if self.last_login_method.startswith("slave_local"): return _parse_result(self.session.local_logout(self._session)) else: diff --git a/scripts/examples/python/exportimport.py b/scripts/examples/python/exportimport.py index bc72580659b..baa1b6ae57d 100755 --- a/scripts/examples/python/exportimport.py +++ b/scripts/examples/python/exportimport.py @@ -19,7 +19,7 @@ # - import raw disk images # - connect an export to an import to copy a raw disk image -from __future__ import print_function + import sys, os, socket, urllib.request, urllib.error, urllib.parse, XenAPI, traceback, ssl, time def exportimport(url, xapi, session, src_vdi, dst_vdi): @@ -63,7 +63,7 @@ def exportimport(url, xapi, session, src_vdi, dst_vdi): ] print("Sending HTTP request:") for h in headers: - output.send("%s\r\n" % h) + output.send((h + "\r\n").encode()) print("%s\r\n" % h) result = output.recv(1024) print("Received HTTP response:") @@ -73,7 +73,7 @@ def exportimport(url, xapi, session, src_vdi, dst_vdi): return # Copy the raw bytes, signal completion by closing the socket - virtual_size = long(xapi.xenapi.VDI.get_virtual_size(src_vdi)) + virtual_size = int(xapi.xenapi.VDI.get_virtual_size(src_vdi)) print("Copying %Ld bytes" % virtual_size) left = virtual_size while left > 0: diff --git a/scripts/examples/python/mini-xenrt.py b/scripts/examples/python/mini-xenrt.py index 0907132da80..b30e9d9973c 100644 --- a/scripts/examples/python/mini-xenrt.py +++ b/scripts/examples/python/mini-xenrt.py @@ -109,7 +109,7 @@ def make_operation_list(vm): print(" -- performs parallel operations on VMs with the specified other-config key") sys.exit(1) - x = xmlrpc.client.server(sys.argv[1]) + x = xmlrpc.client.ServerProxy(sys.argv[1]) key = sys.argv[2] session = x.session.login_with_password("root", "xenroot", "1.0", "xen-api-scripts-minixenrt.py")["Value"] vms = x.VM.get_all_records(session)["Value"] diff --git a/scripts/examples/python/monitor-unwanted-domains.py b/scripts/examples/python/monitor-unwanted-domains.py index 317725288e2..433ea52688e 100644 --- a/scripts/examples/python/monitor-unwanted-domains.py +++ b/scripts/examples/python/monitor-unwanted-domains.py @@ -1,17 +1,37 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python +""" +Script which monitors the domains running on a host, looks for +paused domains which don't correspond to VMs which are running here +or are about to run here, logs them and optionally destroys them. +""" +# DONE: PYTYPE_PASSES, MYPY_PASSES +# TODO: NEEDS_UNIT_TESTS, PYLINT_WARNINGS_IN_BAD_SHAPE, CODE_SMELLS, ADD_DOC_STRINGS +import sys +import time +from subprocess import PIPE, Popen -from __future__ import print_function -import os, subprocess, XenAPI, inventory, time, sys +import inventory # type: ignore[import-untyped] # pylint: disable=import-error +import XenAPI # type: ignore[import-untyped] -# Script which monitors the domains running on a host, looks for -# paused domains which don't correspond to VMs which are running here -# or are about to run here, logs them and optionally destroys them. +# pylint:disable=import-outside-toplevel + +if sys.version_info >= (3,): + from typing import Dict, List, Tuple + # Using simple type aliases here: mypy, pytype and pyright understand them, + # and the code can still be parsed by Python2 (and ignored) + DomID = str + UUID = str + DomIdUUIDTuple = Tuple[DomID, UUID] # Return a list of (domid, uuid) tuples, one per paused domain on this host -def list_paused_domains(): - results = [] - all = subprocess.Popen(["@OPTDIR@/bin/list_domains"], stdout=subprocess.PIPE).communicate()[0] - lines = all.split("\n") +def list_paused_domains(): # type:() -> List[DomIdUUIDTuple] + results = [] # type: List[DomIdUUIDTuple] + all_domains = Popen( + ["@OPTDIR@/bin/list_domains"], + stdout=PIPE, + universal_newlines=True + ).communicate()[0] + lines = all_domains.split("\n") for domain in lines[1:]: bits = domain.split() if bits != []: @@ -27,21 +47,23 @@ def list_paused_domains(): def should_domain_be_somewhere_else(localhost_uuid, domain): (domid, uuid) = domain try: - x = XenAPI.xapi_local() + x = XenAPI.xapi_local() # pytype: disable=name-error + assert x.xenapi x.xenapi.login_with_password("root", "", "1.0", "xen-api-scripts-monitor-unwanted-domains.py") try: try: + assert x.xenapi.VM vm = x.xenapi.VM.get_by_uuid(uuid) resident_on = x.xenapi.VM.get_resident_on(vm) current_operations = x.xenapi.VM.get_current_operations(vm) result = current_operations == {} and resident_on != localhost_uuid if result: - log("domid %s uuid %s: is not being operated on and is not resident here" % (domid, uuid)) + print("domid %s uuid %s: is not being operated on and is not resident here" % (domid, uuid)) return result - except XenAPI.Failure as e: + except XenAPI.Failure as e: # pytype: disable=name-error # XenAPI is not typed yet if e.details[0] == "UUID_INVALID": # VM is totally bogus - log("domid %s uuid %s: is not in the xapi database" % (domid, uuid)) + print("domid %s uuid %s: is not in the xapi database" % (domid, uuid)) return True # fail safe for now return False @@ -50,40 +72,47 @@ def should_domain_be_somewhere_else(localhost_uuid, domain): except: return False -def log(str): - print(str) - # Destroy the given domain def destroy_domain(domain): + # pytype forgets that we import subprocess/Popen on the toplevel, allow it to work: + from subprocess import PIPE, Popen # pylint: disable=reimported,redefined-outer-name + (domid, uuid) = domain - log("destroying domid %s uuid %s" % (domid, uuid)) - all = subprocess.Popen(["@OPTDIR@/debug/destroy_domain", "-domid", domid], stdout=subprocess.PIPE).communicate()[0] + print("destroying domid %s uuid %s" % (domid, uuid)) + Popen(["@OPTDIR@/debug/destroy_domain", "-domid", domid], stdout=PIPE).communicate()[0] # Keep track of when a domain first looked like it should be here -domain_first_noticed = {} +domain_first_noticed = {} # type: Dict[DomIdUUIDTuple, float] # Number of seconds after which we conclude that a domain really shouldn't be here -threshold = 60 +THRESHOLD = 60 if __name__ == "__main__": localhost_uuid = inventory.get_localhost_uuid () while True: time.sleep(1) paused = list_paused_domains () - # GC the domain_first_noticed map - for d in domain_first_noticed.keys(): - if d not in paused: - log("domid %s uuid %s: looks ok now, forgetting about it" % d) - del domain_first_noticed[d] + # The reason was that this loop needs to populate domain_first_noticed, + # (it is initially an empty dict, so the order in the loop is better like this:) + + # populate domain_first_noticed and destroy domains in bad state over threshold: for d in list_paused_domains(): if should_domain_be_somewhere_else(localhost_uuid, d): if d not in domain_first_noticed: domain_first_noticed[d] = time.time() noticed_for = time.time() - domain_first_noticed[d] - if noticed_for > threshold: - log("domid %s uuid %s: has been in bad state for over threshold" % d) + if noticed_for > THRESHOLD: + print("domid %s uuid %s: has been in bad state for over threshold" % d) if "-destroy" in sys.argv: destroy_domain(d) + # Then, garbage-collect domain_first_noticed: + # GC the domain_first_noticed map + # On pylint disable: This needs to iterate of the keys()->list[DomIdUUIDTuple], + # not the item(s), which is time(): domain_first_noticed[d] = time.time() + for dom_id_uuid_tuple in domain_first_noticed: # pylint: disable=consider-using-dict-items + if dom_id_uuid_tuple not in paused: + print("domid %s uuid %s: looks ok now, forgetting about it" % dom_id_uuid_tuple) + del domain_first_noticed[dom_id_uuid_tuple] diff --git a/scripts/examples/python/provision.py b/scripts/examples/python/provision.py index b8aa3f3935f..4c5ab11daef 100644 --- a/scripts/examples/python/provision.py +++ b/scripts/examples/python/provision.py @@ -62,7 +62,7 @@ def setSR(self, sr): def parseProvisionSpec(txt): """Return an instance of type ProvisionSpec given XML text""" - doc = xml.dom.minidom.parseString(txt) + doc = xml.dom.minidom.parseString(txt) # pytype: disable=pyi-error all = doc.getElementsByTagName("provision") if len(all) != 1: raise ValueError("Expected to find exactly one element") @@ -74,7 +74,7 @@ def parseProvisionSpec(txt): def printProvisionSpec(ps): """Return a string containing pretty-printed XML corresponding to the supplied provisioning spec""" - doc = xml.dom.minidom.Document() + doc = xml.dom.minidom.Document() # pytype: disable=pyi-error doc.appendChild(ps.toElement(doc)) return doc.toprettyxml() diff --git a/scripts/examples/python/shell.py b/scripts/examples/python/shell.py index 6e5e4f8ff27..100ed35ac51 100644 --- a/scripts/examples/python/shell.py +++ b/scripts/examples/python/shell.py @@ -103,10 +103,10 @@ def munge_types (str): # We want to support directly executing the cmd line, # where appropriate if len(sys.argv) > cmdAt: - cmd = sys.argv[cmdAt] + command = sys.argv[cmdAt] params = [munge_types(x) for x in sys.argv[(cmdAt + 1):]] try: - print(session.xenapi_request(cmd, tuple(params)), file=sys.stdout) + print(session.xenapi_request(command, tuple(params)), file=sys.stdout) except XenAPI.Failure as x: print(x, file=sys.stderr) sys.exit(2) diff --git a/scripts/examples/smapiv2.py b/scripts/examples/smapiv2.py index cc990dcadf2..b6f5450add9 100644 --- a/scripts/examples/smapiv2.py +++ b/scripts/examples/smapiv2.py @@ -13,7 +13,7 @@ def reopenlog(log_file): if log_file: try: log_f = open(log_file, "a") - except FilenotFoundError: + except IOError: log_f = open(log_file, "w") else: log_f = open(os.dup(sys.stdout.fileno()), "a") diff --git a/scripts/hatests b/scripts/hatests index 8828820ecb3..594c115e7b6 100755 --- a/scripts/hatests +++ b/scripts/hatests @@ -1,15 +1,15 @@ #!/usr/bin/env python -from __future__ import print_function + import XenAPI import getopt import sys import os -import commands +import subprocess import random import time -import httplib -import urllib +import http.client +import urllib.request, urllib.parse, urllib.error def check(svm, ip): """ @@ -20,7 +20,7 @@ def check(svm, ip): global hosts global vmrunning flag = True - masterref2 = svm.xenapi.pool.get_all_records().values()[0]['master'] + masterref2 = list(svm.xenapi.pool.get_all_records().values())[0]['master'] if masterref2 != masterref : print("From " + ip + " point of view the pool master is " + svm.xenapi.host.get_record(masterref2)["address"]) flag = False @@ -28,18 +28,18 @@ def check(svm, ip): if len(hosts) != len(hosts2) : print("From " + ip + " point of view the number of hosts is changed.") flag = False - for k in hosts.keys() : + for k in list(hosts.keys()) : if k not in hosts2 : print("From " + ip + " point of view " + hosts[k]["address"] + " is not present any more.") vmrecords2 = svm.xenapi.VM.get_all_records() vmrunning2 = {} - for k, v in vmrecords2.iteritems() : + for k, v in list(vmrecords2.items()) : if v['power_state'] == 'Running' and int(v['domid']) == 0: vmrunning2[k] = v if len(vmrunning) != len(vmrunning2) : print("From " + ip + " point of view some VMs have changed state.") flag = False - for k, v in vmrunning.iteritems() : + for k, v in list(vmrunning.items()) : if k not in vmrunning2 : print("From " + ip + " point of view " + v['name_label'] + " is not online any more.") if flag : @@ -95,9 +95,9 @@ s.login_with_password('root', 'xenroot', '1.0', 'xen-api-scripts-hatest') slaves = [] master = None vmrecords = s.xenapi.VM.get_all_records() -for k, v in vmrecords.iteritems() : +for k, v in vmrecords.items() : if v['power_state'] == 'Running' and int(v['domid']) > 0: - ip = commands.getoutput("xenstore-ls /local/domain/" + v['domid'] + " | grep ip") + ip = subprocess.getoutput("xenstore-ls /local/domain/" + v['domid'] + " | grep ip") try: ip = ip.split()[2] ip = ip[1:-1] @@ -109,14 +109,14 @@ for k, v in vmrecords.iteritems() : svm = XenAPI.Session("http://" + slaves[0][1]) try : svm.login_with_password('root', 'xenroot', '1.0', 'xen-api-scripts-hatest') - masterref = svm.xenapi.pool.get_all_records().values()[0]['master'] + masterref = list(svm.xenapi.pool.get_all_records().values())[0]['master'] masterrecord = svm.xenapi.host.get_record(masterref) masterip = masterrecord['address'] except XenAPI.Failure as inst: masterip = inst.details[1] svm = XenAPI.Session("http://" + masterip) svm.login_with_password('root', 'xenroot', '1.0', 'xen-api-scripts-hatest') - masterref = svm.xenapi.pool.get_all_records().values()[0]['master'] + masterref = list(svm.xenapi.pool.get_all_records().values())[0]['master'] for i in slaves : if masterip == i[1] : master = i @@ -127,13 +127,13 @@ print("Master ip address is " + master[1]) #getting ip -> hostref references hosts = {} hostsrecs = svm.xenapi.host.get_all_records() -for k, v in hostsrecs.iteritems() : +for k, v in hostsrecs.items() : hosts[v['address']] = k #getting the VM running vmrunning = {} vmrecords = svm.xenapi.VM.get_all_records() -for k, v in vmrecords.iteritems() : +for k, v in vmrecords.items() : if v['power_state'] == 'Running' and int(v['domid']) == 0: vmrunning[k] = v @@ -160,7 +160,7 @@ elif sys.argv[-1] == "slave_hard_failure" : elif sys.argv[-1] == "master_vif_unplug" : print("Unplugging the first found attached VIF in the master") allvifs = s.xenapi.VIF.get_all_records() - for k, v in allvifs.iteritems() : + for k, v in allvifs.items() : if v['currently_attached'] and v['VM'] == master[0]: vifbringup = k s.xenapi.VIF.unplug(vifbringup) @@ -195,7 +195,7 @@ sys.stdout.write("\n") print("Collecting logs now...") try : fileout = open("master-" + master[1] + "-log.tar.bz2", "w") - f = urllib.urlopen("http://root:xenroot@" + master[1] + "/system-status?host_id=" + hosts[master[1]]) + f = urllib.request.urlopen("http://root:xenroot@" + master[1] + "/system-status?host_id=" + hosts[master[1]]) buf = f.read(50) if len(buf) == 0 : print(master[1] + " returned an empty log.") @@ -215,7 +215,7 @@ except: for k, ip in slaves : try : fileout = open("slave-" + ip + "-log.tar.bz2", "w") - f = urllib.urlopen("http://root:xenroot@" + ip + "/system-status?host_id=" + hosts[ip]) + f = urllib.request.urlopen("http://root:xenroot@" + ip + "/system-status?host_id=" + hosts[ip]) buf = f.read(50) if len(buf) == 0 : print(ip + " returned an empty log.") diff --git a/scripts/hfx_filename b/scripts/hfx_filename index e454eb299e5..beba497cbec 100755 --- a/scripts/hfx_filename +++ b/scripts/hfx_filename @@ -33,9 +33,9 @@ def rpc(session_id, request): ] #print "Sending HTTP request:" for h in headers: - s.send("%s\r\n" % h) + s.send((h + "\r\n").encode()) #print "%s\r\n" % h, - s.send(request) + s.send(request.encode()) result = s.recv(1024) #print "Received HTTP response:" @@ -43,10 +43,10 @@ def rpc(session_id, request): if "200 OK" not in result: print("Expected an HTTP 200, got %s" % result, file=sys.stderr) return - id = result.find("\r\n\r\n") + id = result.find(b"\r\n\r\n") headers = result[:id] - for line in headers.split("\r\n"): - cl = "content-length:" + for line in headers.split(b"\r\n"): + cl = b"content-length:" if line.lower().startswith(cl): length = int(line[len(cl):].strip()) body = result[id+4:] @@ -55,6 +55,7 @@ def rpc(session_id, request): s.close() def parse_string(txt): + assert isinstance(txt, str) prefix = "success" if not txt.startswith(prefix): raise "Unable to parse string response" diff --git a/scripts/mail-alarm b/scripts/mail-alarm index a0e32283044..13efb81c4fb 100755 --- a/scripts/mail-alarm +++ b/scripts/mail-alarm @@ -15,6 +15,7 @@ from __future__ import print_function import json import os import re +import subprocess import sys import syslog import tempfile @@ -113,6 +114,7 @@ def get_config_file(): def load_mail_language(mail_language): + mail_language_file = "" try: mail_language_file = os.path.join( mail_language_pack_path, mail_language + ".json" @@ -1024,13 +1026,18 @@ def main(): # Write out a temporary file containing the new config fd, fname = tempfile.mkstemp(prefix="mail-", dir="/tmp") try: - os.write(fd, config) + os.write(fd, config.encode()) os.close(fd) # Run ssmtp to send mail - chld_stdin, chld_stdout = os.popen2( - ["/usr/sbin/ssmtp", "-C%s" % fname, destination] + + proc = subprocess.Popen( + ["/usr/sbin/ssmtp", "-C%s" % fname, destination], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE ) + chld_stdin = proc.stdin + assert chld_stdin chld_stdin.write("From: %s\n" % sender) chld_stdin.write('Content-Type: text/plain; charset="%s"\n' % charset) chld_stdin.write("To: %s\n" % destination.encode(charset)) @@ -1040,6 +1047,9 @@ def main(): chld_stdin.write("\n") chld_stdin.write(msg.generate_email_body().encode(charset)) chld_stdin.close() + proc.communicate() + chld_stdout = proc.stdout + assert chld_stdout chld_stdout.close() os.wait() diff --git a/scripts/nbd_client_manager.py b/scripts/nbd_client_manager.py index bebe97a2587..2b76386db0f 100644 --- a/scripts/nbd_client_manager.py +++ b/scripts/nbd_client_manager.py @@ -149,7 +149,9 @@ def _get_persistent_connect_info_filename(device): the connection details. This is based on the device name, so /dev/nbd0 -> /var/run/nonpersistent/nbd/0 """ - number = re.search('/dev/nbd([0-9]+)', device).group(1) + match = re.search('/dev/nbd([0-9]+)', device) + assert match + number = match.group(1) return PERSISTENT_INFO_DIR + '/' + number def _persist_connect_info(device, path, exportname): @@ -223,7 +225,7 @@ def disconnect_nbd_device(nbd_device): def _connect_cli(args): device = connect_nbd(path=args.path, exportname=args.exportname) - print device + print(device) def _disconnect_cli(args): diff --git a/scripts/perfmon b/scripts/perfmon index 1af3ce05772..e3dc58f7563 100644 --- a/scripts/perfmon +++ b/scripts/perfmon @@ -380,7 +380,13 @@ def get_percent_mem_usage(ignored): memlist = memfd.readlines() memfd.close() memdict = [ m.split(':', 1) for m in memlist ] - memdict = dict([(k.strip(), float(re.search('\d+', v.strip()).group(0))) for (k,v) in memdict]) + new_memdict = {} + for k, v in memdict.items(): + key = k.strip() + match = re.search('\d+', v.strip()) + assert match + new_memdict[key] = float(match.group(0)) + memdict = new_memdict # We consider the sum of res memory and swap in use as the hard demand # of mem usage, it is bad if this number is beyond the physical mem, as # in such case swapping is obligatory rather than voluntary, hence diff --git a/scripts/plugins/extauth-hook-AD.py b/scripts/plugins/extauth-hook-AD.py index 98b228c04e5..314ea2e1083 100755 --- a/scripts/plugins/extauth-hook-AD.py +++ b/scripts/plugins/extauth-hook-AD.py @@ -257,6 +257,7 @@ def _match_subject(self, subject_rec): def _add_upn(self, subject_rec): sep = "@" + upn = "" try: upn = subject_rec["other_config"]["subject-upn"] user, domain = upn.split(sep) diff --git a/scripts/restore-sr-metadata.py b/scripts/restore-sr-metadata.py index 105591a15c5..fb44557b78e 100644 --- a/scripts/restore-sr-metadata.py +++ b/scripts/restore-sr-metadata.py @@ -2,25 +2,20 @@ # Restore SR metadata and VDI names from an XML file # (c) Anil Madhavapeddy, Citrix Systems Inc, 2008 -import atexit +# pytype: disable=pyi-error import XenAPI import os, sys, time import getopt from xml.dom.minidom import parse + import codecs sys.stdout = codecs.getwriter("utf-8")(sys.stdout) sys.stderr = codecs.getwriter("utf-8")(sys.stderr) -def logout(): - try: - session.xenapi.session.logout() - except: - pass -atexit.register(logout) def usage(): - print >> sys.stderr, "%s -f -u " % sys.argv[0] + print("%s -f -u " % sys.argv[0], file=sys.stderr) sys.exit(1) def main(argv): @@ -29,8 +24,8 @@ def main(argv): try: opts, args = getopt.getopt(argv, "hf:u:", []) - except getopt.GetoptError, err: - print str(err) + except getopt.GetoptError as err: + print(str(err)) usage() infile = None @@ -47,11 +42,11 @@ def main(argv): try: doc = parse(infile) except: - print >> sys.stderr, "Error parsing %s" % infile + print("Error parsing %s" % infile, file=sys.stderr) sys.exit(1) if doc.documentElement.tagName != "meta": - print >> sys.stderr, "Unexpected root element while parsing %s" % infile + print("Unexpected root element while parsing %s" % infile, file=sys.stderr) sys.exit(1) for srxml in doc.documentElement.childNodes: @@ -60,19 +55,19 @@ def main(argv): name_label = srxml.getAttribute("name_label") name_descr = srxml.getAttribute("name_description") except: - print >> sys.stderr, "Error parsing SR tag" + print("Error parsing SR tag", file=sys.stderr) continue # only set attributes on the selected SR passed in on cmd line if sruuid is None or sruuid == "all" or sruuid == uuid: try: srref = session.xenapi.SR.get_by_uuid(uuid) - print "Setting SR (%s):" % uuid + print("Setting SR (%s):" % uuid) session.xenapi.SR.set_name_label(srref, name_label) - print " Name: %s " % name_label + print(" Name: %s " % name_label) session.xenapi.SR.set_name_description(srref, name_descr) - print " Description: %s" % name_descr + print(" Description: %s" % name_descr) except: - print >> sys.stderr, "Error setting SR data for: %s (%s)" % (uuid, name_label) + print("Error setting SR data for: %s (%s)" % (uuid, name_label), file=sys.stderr) sys.exit(1) # go through all the SR VDIs and set the name_label and description for vdixml in srxml.childNodes: @@ -81,17 +76,17 @@ def main(argv): vdi_label = vdixml.getAttribute("name_label") vdi_descr = vdixml.getAttribute("name_description") except: - print >> sys.stderr, "Error parsing VDI tag" + print("Error parsing VDI tag", file=sys.stderr) continue try: vdiref = session.xenapi.VDI.get_by_uuid(vdi_uuid) - print "Setting VDI (%s):" % vdi_uuid + print("Setting VDI (%s):" % vdi_uuid) session.xenapi.VDI.set_name_label(vdiref, vdi_label) - print " Name: %s" % vdi_label + print(" Name: %s" % vdi_label) session.xenapi.VDI.set_name_description(vdiref, vdi_descr) - print " Description: %s" % vdi_descr + print(" Description: %s" % vdi_descr) except: - print >> sys.stderr, "Error setting VDI data for: %s (%s)" % (vdi_uuid, name_label) + print("Error setting VDI data for: %s (%s)" % (vdi_uuid, name_label), file=sys.stderr) continue if __name__ == "__main__": diff --git a/scripts/static-vdis b/scripts/static-vdis index 77c9790b71e..60c1adf35a1 100755 --- a/scripts/static-vdis +++ b/scripts/static-vdis @@ -78,7 +78,7 @@ def check_clusterstack(ty): def sr_attach(ty, device_config): check_clusterstack(ty) - + assert device_config args = [arg for (k,v) in device_config.items() for arg in ["--configuration", k, v]] return call_volume_plugin(ty, "SR.attach", args) @@ -239,6 +239,7 @@ def call_backend_attach(driver, config): def call_backend_detach(driver, config): params = xmlrpc.client.loads(config)[0][0] + assert params params['command'] = 'vdi_detach_from_config' config = xmlrpc.client.dumps(tuple([params]), params['command']) xml = doexec([ driver, config ]) diff --git a/scripts/static-vdisc b/scripts/static-vdisc new file mode 100644 index 0000000000000000000000000000000000000000..b593ac5d495adfdd8c28d746ba49606150649d7c GIT binary patch literal 15505 zcmcgzTWnm%d7iV&+ftNB@ha+Ktz=zjOUsqzL`@vUu`Eiq8k><1ZPQ>ZZB~zd>nxsXMmo!1zw9P}>hoV3qdI8$>f+8*2pl^9>AM#qD2wI@;ZNKlEvo}U* zjG`>bot-mhuK)c1_g`k1{*U9sfApQ-4NE5Z>Bsjg_}EXTjS2AQnTjzh?Yw7JyjFS2 ztfX4`v{^~F@_lBdzm*^8sy}E}23zGrW@V_A&zO}=D?e;jj+p>GkC*^`9ybAcJ7I!; zvodOe0rMdyGiHK8Wlox4NSSdHWR#gO!LTxuCOD?dlnF+ZnKr?3WoArpLYY%07*%H0 z1Y^paHo-|{&X{0anX{xoFkwE#!k;n0qzccQU`m;+38s~~V1gNCE}Gz!GS8Y|R+&pC zIIYZO6P!`ziV4mtbJc`%COBudQYQL%I(*KA#%w-so?ygh%$70HuX*8B^TZghW|DWo zc{3Yi)#AtA;TGuT$KLK2z3>zpEj+qvR<4=dSreW%;Wc)<@aULXxo*Pi=C#+if9Nd- z*O>WhZ+RQTYi#v@LH##UJ^f#76%Lur9Q$9HH{rb5%$v;{CctvBP%QC^dD3URFJ3g^ z8NN<>ea6ftxvOUON!nm&uhsemt?<3xDa>iKKj@vp)mGuzu70sBuea@Us`hVBYm36m zy=_-+n(!v6&_jXe)y1{mJ#oXM@JeqN8os?xSlH6sb%;>;zwonYPa~TT<5GS%tl4@q zDusC)7vpj%w-c1D&GL2AM)^v4HQWtRt=K5%Y+R3u>tW88qH-g)$5AN=cgm$OSE|?6 z%IkKH&31}WK5EwTwR)`)Mz(C@uoma5%}QKu6yuFTSSzkp!k`3EHvRbY;d2)s`!v#C<6B4v^_Ln*(N_ zk61THS+wVn=6C9qW;M*$qxE^Le4fjiPnI0?JdWpPt-OP^6f1>V__$K8 zg?4Tb#qk6Z_EAXsC^*u@11Jx7vG5=sM%Ff08&SO!S{skxv9uAEwhHxT+-Sy_vN3N* z%}|+5Td!g6##HLXz-r!-heljp7ONEoN9ZIU&X^u-D7O(%t6XvA2w6mVeV*JJDTR!ZuWDVyLOa= z<%42eBwyuyJekA0=W|czNEbiFgcI>eU|(e+$Yd`LKX2^y#XO6jW+A$T1+}cVKI;-h+l0C zNSNas)!$mV0x<{Aa^`DCO2g4)E``S%^-5Ua$QMv#Ia%YK@=Bak8lN`t@_2X(@g@y; z4_xPn2eLsOvFJgwmpY0)YteI4Z~C9LMu#SVraoIuZk|e+myrN{lfK zpc!;g<1)%R)Zm7Y66E~5eDg0z%HS8V(k<}M?&lMZ0*$~+Fmhnwz_wW*grwU##1z-EWSn|+CdM$eHNPFlfw zSU5b)IXn~=CMC_N-2J!}?x`tc2!O9HFWbESRLwm=5pPN6l%ZtaFIH@5r;$r)gq1K3 zb2X@%+-kiZ3)C!`AUO19j?<1$AhA{8{mzR8H*N7}6>6anlp`&%ZZ#$$HBn>G;MSjI zdrAQ*qE-n2Y&mo#BuG92K(;Yhp??M6K4noQL2dO&S)X&G)_e}%mP4D=aqprx;f;GU zUfLV;j_bP*sws`!Fw#DxCDvqHe(vI9BP4B9g#fJgnFFlB5L^!&Hy18ZQ1aQffnC{k1@zXZ~KhF-*P}m<_0Z_!n>tPIi>;%%Ek2> z_M=>~(gwL4?SrSPIGGt`Ti9;GM2kZRG>}H>>>_eMRqbhK)SaD`}a{>Ss7ta=XDM4|o}<%2E6d zbQETLQ$K-99n6PTOiNW_3$>BZpV;XqDYG?TqK|1u(J8D;Z4GI%Xi18PvUKL(YKGBX z0_|=_F@-V;x0~fC3<{-66IO2otFTzw@@W`3YEu5R#f$Y?=)Zs`{}x|LZz|dI&$9rZ zqqW{-fhN<`EFWOR#N-%HC0dhuf@_ypyn~FRAUSs(;19}qG%OY#&6)RMq02~z4MvuEdkSxm>qVqjJqQxG3RZgf}8i}_(174-f(Ib zkR-_AyxQDJCICeNGkX&xGUy5_5jIS>GQ$GEJyfh7fOG#6 z3pfZ1w0cOZpF@F~IeJtoM@|Y3U!Y(4E{2vI8KLlmFj(Xu^S8;-#6v=krNiXd{WPT$ zT!w{_3^IRbK#he*7b()D4!E*~xTvx`h{5+Zw${v~G@a|(-P zBs7UYJSI2f>iMLu|0)xal`I$)YrxJlaTHONuon?3ruhmcaNa;+kWi1Nr zMuB$s2Y9lRNF1R@y{tE)-!znBiK}YKHfL}9QF2)2fw2ANt8i~$BtaJ*U88^#0K|A- z0bTn{blxD-E}TL2?eqLw{xW8R9tO-WgRRphc^uTEj6~vu2A##;0U?M7LIpn4-^V_} ziAOEG8`8TW1Zgz52MdxL3#|Uvu3BJ=;Y2RHPTON@n`yVLP4_k#>1whxM*vyuvGM#Y zDMSn}HKPdoT-nPx!43nF_^_!(%SkL$B2)82UZI0vpF!eQosdlod}^{C>qx**{F1e7 z`%+f2>OVl%v+O zFU?8Uqs^efjc4#x!8bn z95@Vqo`oa|3}EYXtNgc^_$=oxC1}g6&jlXA+GJ(aVSGgJVG0D_3I{ri#2HG*WfPC4 zMpIY3(}0C(?_6ry1#4)=kT!Smu{Qw@yI-YzKcSZYl7eso-&sHcIdgznV0NRR>oN2+q@E%#DMRCj+mVOA<$WAQ zfY*jh^k*Ik1~3!BYEWPRRP2RkZh@%qi*Fs^gaVzS1^Oj+?xxW5K1#>TZP>4yGiD$7 z3B(;U@hKF|8on2#cIXD;pKcNJq>0b44vr_V+WnKnZ5r0&W*=uCdDd*6Gx0MV*A0y$i`%sgv-im^3Y{k{lGm77XJ512589DfEXJ8-u;9As zi!Yc>oD2+`=tn8Knwqisfz9i~Twi$O4F-RAaxJ;%+O>g%q zdtZ2T-W*KH?f@^7$EWZaGx(xj^0tqqxwT7kXCOPhI^^ldLKRG0Fh2EyWmqYHSee*C zmL_om!pV}N^jYV5$hTZWGlnEQV%NP^Mpnm(HK-d~Y47|%J(&@teBI0{Nc0H_x^bruM z;Xvh5LXa%Oe=lV(+c|;tee@@DP1c(_Y}OF8-15)ijfT6oTM8R=HuM(Za70HmZ-#uh z9$0vV)xCnkJ$-oIfJ3g9<=PIcfjS&P8V9UasUw)n_pUbgm&MqEWn@J>K64w|5A4_f zWcTZbuP#{402z7OY!ij>eFL|nZ^rbMSjrf@dt-evip+wbS#4PVSv>l8SVq56#H*7`7Dw->J`05{K}HrLnwetAWAuwV~H3SyQ=Al2&m(i*qGLajeNOz zTr2!5(N5e=b=uJC8zhLUTy@$RG_o#ht9ZF4QDDGbb7)hAmWDL6i-E|N7L&>ZQE~*j zni5hBf?wof{_fGOR)#pv5ro1zVY~Z(Vz&nfYF_aHLg&+4O5x^Qsjd-J} zv^VGt_l@Cs0uhiY#J$J86W+yi+M9*#Kkl9G%i#GqeoLekRXmybEqEuME2Y8*CPL{03!T%C^BQ#JLG7a|y|6FQ zUV{3Drbl&xu($msvpvSYWhg8AO$td{>qChNPl*b>c>1$Y)93M~EH_K;&LgIqkE+>R zv<7Gp0xlr7dI&^o^%y|fgeAuTW$5_=5OTOf5n8(9vQnjP!w#s}J%oFzegUs_X6Ns; zDyl>OF?+beHiI1wzTJC?ee#}Lsm&^^ z1-9!jkj$^0eAqtOdidbsJsJ9feIB&pFcgp(3WVL1I1Oj$Q#B-NfTy^KzzbJ+%^)yQP(H1kfh)u-)Lio5-uvm5AioL^r zL3CRYCC*&ov_>(qVWGAf6ri;e($s3eSe7X!C;FgTv1(c7u$$8gcN_I6KGM}wB4G<@ zKJH}W1JrZ^wO`5Y=RmTI#ko+!(wl`>@7(?Dy~TyX{W}ld zc*nmtcR9EY2;}EEMY#NLqOt_WZJ@i>pv~ivaOi&>?*(kCv|Fu2jgqoBq=5OO!*l;z zsN|w#k64{t=D)&3QbO)SL#cm!7tn)v7jHF?ObZdiDVNnSTYZ&*jb%>IyULNK%v;Azr`kE z4jop|_zFS>)(dMyoX+($~TUuf-%_BvYS>y5bU=2h#G6+zaC6^_+vxM>y5H})hp3B*}4W-iAF zqr&L9oP$^z=T5niYY(6H(e`#Bt1HL}tvk(*lGgq=Fy4RSBR`L32!BVMUTT@4BIP+w zOfuF@N}Rlc+Z7Gc`Pu)1c*E#eKCV=Rv|}#kF0DC|kYFN-yOhP-I4-2hmfVn}GM|fEL`X1MGk`!IT$( z7O=pf_Bt;RTIX@LqU$5kx=eh<*cq^MIK=kQQ3NgI(140j4bD?DN)6&d$I=`L=dQ}U zq@(H%uYI~qY72r8odVst4$xSZ6O66hG7Ib_B(K33hl5?Ll!NTzs|(q8{0EA1AWV?} zI>N`=JGwHqffI9u$hDOS2}H4dj2muUCi&-4&v}S^88h0_nOUI$$FRIB^TyVpFyaD* z%>O;qI#LPwbqsKfyUZAZ(c$$tT((IX>PWTYv~h^4+VN!&4w8b6-u-wATkwR=+Vd%A zgi{}Y9lD)0r=RG3y_BNkmR67zoFK<-Pw^~wu@1Yd@NC!ZnE(U9*oQsiVy><|%wATs z-=pAAgSogabWQT?!n!a~K+kr34jt9L;3YMN)B~c5h)y8tnv`O0N#E!mQQ4%}4R?TB zLvFZX*Lb+M@iCRM;W5TE5Xjv`Y)%19gl!p@JqF$4Ay|X(1_Pxj^EIeTE)Kyz6u~A& z6mJ>dZ@ovoJEfo|LeLN>Oc4qffn3CGhZ#mqBAq8pjEmd4uZ`d=0;#hLm;wIYN25`L zkZaNo5z?-99>STpkxe`BpFIXHVHA=+uQ1MuXel@FOo}fMTRcFR^%SG4Nw>I;f;HfF zI0O~~jd_iO3sP*4Sz|ef&q^S*p;!{P2z$HK%#;TX*c*p0EqfL#` z7ngKVNSAl9BIj@S-h-kMm|VQg@mQq$xpgtBb3a%yn4+!&EF@PulM4X; zYv`y%3?xiQ?(uDTo(-md^lZnWaac$43atQBK&E%SZSk+W{U}j?715uq1zlkV;c&kS z(L;gK;%czy?i>6uiv2%f@~21~7I1jEcJnv<4D)PTt~QSJBwd#wZZq(?+lXHNIbsHf z#kH4|;(;7LtimLBS0@xs?hp`{+;-r~`n}8X35o)_pB)q-V0W6&o#o?2Cij^PFuBC! zGLsLOFsdm--@nS-vrNch{&$$rjFPb^Z=xAhUTFEdeW=6UAC>+)S_pvw7mjGWxc zRY_0aU7e#&fx&AohO-J?0YzP)P1^wnOFMa(TK|{0 zO+jiSOb`}Z81ov+d-2=3SPD@%?LqYTA49Mb(x<`8AEKrI`%DBhlG&?#?7+)+`S|Bd zzQ^P*m`D~05e}LJ1Y}l4Y7YZ}=+_1U|8H3D`%M0p3A;K>; z!+5Aybm?yOv`hPd_{hm}3sYU+65Cz~eat$B1cb=QBn_p)YqJV9FlcOQ3@!7*D` zSXRt@1ZP{c(3<3ckZeqyXS{xC6l#?*Pv>t1ggS8hyda0rg(S(Z{)b59E^5LNFSpZ0 zvRu>)^EYsl8bL+(&r7ZUN5lW8@hDjT!6r{qi^pF2RD1s4BXM&3A|L;Oi7q67}%P1dzc0wl5sTugL6iHClh?=CTs!SpP1G*{fIs}M*w&^eaT zbkz-D|5MD}VM2Cx2dD2cM>_alVIp(t>&$(N$#KzABWv9*tR5IumGD934x z%qB-ylsO6?0W%Gu@7vXS(5!?nb8x%C0@xrU-lfz4>=S%anc+--rVqc<$PHu$GeemZ znTgCoW^`l-Z%*QA5alD87c%EEnGrzdaB2>mIDx~G5e0;2VFO)EokV^Tp9z2|a;g6Y Di!#i3 literal 0 HcmV?d00001 diff --git a/scripts/test_mail-alarm.py b/scripts/test_mail-alarm.py index 2a918f5edbe..b309984b1fe 100644 --- a/scripts/test_mail-alarm.py +++ b/scripts/test_mail-alarm.py @@ -7,6 +7,8 @@ import shutil import sys import unittest +from scripts.unit_tests.import_helper import import_file_as_module + import mock def nottest(obj): @@ -63,10 +65,7 @@ def setUp(self): try: self.work_dir = tempfile.mkdtemp(prefix="test-mail-alarm-") log_file_global = os.path.join(self.work_dir, "user.log") - src_file = "./scripts/mail-alarm" - dst_file = os.path.join(self.work_dir, "mailalarm.py") - shutil.copyfile(src_file, dst_file) - sys.path.append(self.work_dir) + self.mail_alarm = import_file_as_module("scripts/mail-alarm") except: raise @@ -83,7 +82,7 @@ def common_test_good_input( body_str, xmlbody_str=XML_BODY_COMMON, ): - import mailalarm + mailalarm = self.mail_alarm # Emulate functions with Mock mock_setup(mailalarm) @@ -111,7 +110,7 @@ def common_test_bad_input( xmlbody_str=XML_BODY_COMMON, ): global log_file_global - import mailalarm + mailalarm = self.mail_alarm # Emulate functions with Mock mock_setup(mailalarm) diff --git a/scripts/test_static_vdis.py b/scripts/test_static_vdis.py index b0ab6ad5939..e8bdbea05ea 100644 --- a/scripts/test_static_vdis.py +++ b/scripts/test_static_vdis.py @@ -1,56 +1,28 @@ -#!/usr/bin/env python3 -# -# unittest for static-vdis - -import unittest -from mock import MagicMock +"""unittest for static-vdis""" import sys -import os -import subprocess import tempfile +import unittest -# mock modules to avoid dependencies -sys.modules["XenAPI"] = MagicMock() -sys.modules["inventory"] = MagicMock() - -def import_from_file(module_name, file_path): - """Import a file as a module""" - if sys.version_info.major == 2: - return None - else: - from importlib import machinery, util - loader = machinery.SourceFileLoader(module_name, file_path) - spec = util.spec_from_loader(module_name, loader) - assert spec - assert spec.loader - module = util.module_from_spec(spec) - # Probably a good idea to add manually imported module stored in sys.modules - sys.modules[module_name] = module - spec.loader.exec_module(module) - return module - -def get_module(): - """Import the static-vdis script as a module for executing unit tests on functions""" - testdir = os.path.dirname(__file__) - return import_from_file("static_vdis", testdir + "/static-vdis") +from .unit_tests.import_helper import import_file_as_module, mocked_modules -static_vdis = get_module() -@unittest.skipIf(sys.version_info < (3, 0), reason="requires python3") +# @unittest.skipIf(sys.version_info < (3, 0), reason="script migrated to python3") class TestReadWriteFile(unittest.TestCase): def test_write_and_read_whole_file(self): """Test read_whole_file and write_whole_file""" - test_file = tempfile.NamedTemporaryFile(delete=True) - filename = str(test_file.name) + content = r"""def read_whole_file(filename): with open(filename, 'r', encoding='utf-8') as f: return ''.join(f.readlines()).strip() - def write_whole_file(filename, contents): with open(filename, "w", encoding='utf-8') as f: f.write(contents)""" - static_vdis.write_whole_file(filename, content) - expected_content = static_vdis.read_whole_file(filename) - self.assertEqual(expected_content, content) - - \ No newline at end of file + + # Mock and XenAPI and inventory for importing scripts/static-vdis: + with mocked_modules("XenAPI", "inventory", "xmlrpc", "xmlrpc.client", "urllib.parse"): + static_vdis = import_file_as_module("scripts/static-vdis") + + # Test the update write_whole_file and read_whole_file functions: + with tempfile.NamedTemporaryFile() as temp: + static_vdis.write_whole_file(temp.name, content) + assert static_vdis.read_whole_file(temp.name) == content diff --git a/scripts/test_usb_scan.py b/scripts/test_usb_scan.py index c64d89d8276..365060533d7 100644 --- a/scripts/test_usb_scan.py +++ b/scripts/test_usb_scan.py @@ -120,7 +120,7 @@ def tearDown(self): @nottest def test_usb_common(self, moc_devices, moc_interfaces, moc_results, path="./scripts/usb-policy.conf"): - import usb_scan + from . import usb_scan mock_setup(usb_scan, moc_devices, moc_interfaces, path) devices, interfaces = usb_scan.get_usb_info() @@ -136,7 +136,7 @@ def test_usb_exit(self, devices, interfaces, results, with self.assertRaises(SystemExit) as cm: self.test_usb_common(devices, interfaces, results, path) if msg: - self.assertIn(msg, cm.exception.code) + self.assertIn(msg, str(cm.exception)) def test_usb_dongle(self): devices = [ diff --git a/scripts/unit_tests/__init__.py b/scripts/unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/unit_tests/conftest.py b/scripts/unit_tests/conftest.py index fc9a52ea024..dcc2e5a26ad 100644 --- a/scripts/unit_tests/conftest.py +++ b/scripts/unit_tests/conftest.py @@ -23,7 +23,8 @@ def pytest_configure(config): # pylint: disable=unused-argument :param config: The pytest configuration object. """ - sys._called_from_pytest = True # pylint: disable=protected-access + # pylint: disable=protected-access + sys._called_from_pytest = True # type: ignore[attr-defined] def pytest_unconfigure(config): # pylint: disable=unused-argument @@ -35,7 +36,7 @@ def pytest_unconfigure(config): # pylint: disable=unused-argument :param config: The pytest configuration object. """ - del sys._called_from_pytest + del sys._called_from_pytest # type: ignore[attr-defined] @pytest.fixture(scope="function") diff --git a/scripts/unit_tests/import_helper.py b/scripts/unit_tests/import_helper.py new file mode 100644 index 00000000000..32425e5196e --- /dev/null +++ b/scripts/unit_tests/import_helper.py @@ -0,0 +1,74 @@ +"""pytest fixtures for unit-testing functions in the xtf-runner script""" +import os +import sys + +from contextlib import contextmanager +from types import ModuleType + +@contextmanager +def mocked_modules(*module_names): + """Context manager that temporarily mocks the specified modules. + + :param module_names: Variable number of names of the modules to be mocked. + :yields: None + + During the context, the specified modules are added to the sys.modules + dictionary as instances of the ModuleType class. + This effectively mocks the modules, allowing them to be imported and used + within the context. After the context, the mocked modules are removed + from the sys.modules dictionary. + + Example usage: + ```python + with mocked_modules("module1", "module2"): + # Code that uses the mocked modules + ``` + """ + for module_name in module_names: + sys.modules[module_name] = ModuleType(module_name) + yield + for module_name in module_names: + sys.modules.pop(module_name) + + +def import_file_as_module(relative_script_path): + """Import a Python script without the .py extension as a python module. + + :param relative_script_path (str): The relative path of the script to import. + :returns module: The imported module. + :raises: AssertionError: If the spec or loader is not available. + + Note: + - This function uses different methods depending on the Python version. + - For Python 2, it uses the imp module. + - For Python 3, it uses the importlib module. + + Example: + - import_script_as_module('scripts/mail-alarm') # Returns the imported module. + """ + script_path = os.path.dirname(__file__) + "/../../" + relative_script_path + module_name = os.path.basename(script_path) + + if sys.version_info.major == 2: + # Use deprecated imp module because it needs also to run with Python27: + # pylint: disable-next=import-outside-toplevel, deprecated-module + import imp # pyright: ignore[reportMissingImports] + + return imp.load_source(module_name, script_path) + + # For Python 3.11+: Import Python script without the .py extension: + # https://gist.github.com/bernhardkaindl/1aaa04ea925fdc36c40d031491957fd3: + # pylint: disable-next=import-outside-toplevel + from importlib import ( # pylint: disable=no-name-in-module + machinery, + util, + ) + loader = machinery.SourceFileLoader(module_name, script_path) + spec = util.spec_from_loader(module_name, loader) + assert spec + assert spec.loader + module = util.module_from_spec(spec) + # It is probably a good idea to add the imported module to sys.modules: + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module diff --git a/scripts/unit_tests/test_usb_reset.py b/scripts/unit_tests/test_usb_reset.py index c11d6b3f881..04545ed2d42 100644 --- a/scripts/unit_tests/test_usb_reset.py +++ b/scripts/unit_tests/test_usb_reset.py @@ -19,9 +19,6 @@ # For mocking open: builtins vs __builtin__ differs between Python2 and Python3: BUILTIN = "__builtin__" if sys.version_info < (3,) else "builtins" -if sys.version_info >= (3, 0): - pytest.skip(allow_module_level=True) - def assert_error(captured_logs, error_message): """Assert that the captured logs contain exactly the specified error message. @@ -208,11 +205,11 @@ def test_open_unbuffered(usb_reset, tmp_path, mocker): ] mock_open_func.assert_has_calls( [ - mocker.call(tmp + "/tasks", "w", 0), + mocker.call(tmp + "/tasks", "wb", 0), mocker.call().__enter__(), # pylint: disable=unnecessary-dunder-call - mocker.call(tmp + "/devices.deny", "w", 0), + mocker.call(tmp + "/devices.deny", "wb", 0), mocker.call().__enter__(), # pylint: disable=unnecessary-dunder-call - mocker.call(tmp + "/devices.allow", "w", 0), + mocker.call(tmp + "/devices.allow", "wb", 0), mocker.call().__enter__(), # pylint: disable=unnecessary-dunder-call mocker.call().write(b"a"), mocker.call().write(b"c 1:3 rw"), diff --git a/scripts/usb_reset.py b/scripts/usb_reset.py index c0141c84fe1..b89b6a34f88 100755 --- a/scripts/usb_reset.py +++ b/scripts/usb_reset.py @@ -157,7 +157,7 @@ def dev_path(device): exit(1) -def get_ctl(path, mode): +def get_ctl(path, mode): # type:(str, str) -> str """get the string to control device access for cgroup :param path: the device file path :param mode: either "r" or "rw" @@ -200,7 +200,7 @@ def deny_device(path, domid): _device_ctl(path, domid, False) -def setup_cgroup(domid, pid): +def setup_cgroup(domid, pid): # type:(str, str) -> None cg_dir = get_cg_dir(domid) try: @@ -212,17 +212,17 @@ def setup_cgroup(domid, pid): try: # unbuffered write to ensure each one is flushed immediately - with open(cg_dir + "/tasks", "w", 0) as tasks, \ - open(cg_dir + "/devices.deny", "w", 0) as deny, \ - open(cg_dir + "/devices.allow", "w", 0) as allow: + with open(cg_dir + "/tasks", "wb", 0) as tasks, \ + open(cg_dir + "/devices.deny", "wb", 0) as deny, \ + open(cg_dir + "/devices.allow", "wb", 0) as allow: # deny all - deny.write("a") + deny.write(b"a") # grant rw access to /dev/null by default - allow.write(get_ctl("/dev/null", "rw")) + allow.write(get_ctl("/dev/null", "rw").encode()) - tasks.write(str(pid)) + tasks.write(str(pid).encode()) except (IOError, OSError, RuntimeError) as e: log.error("Failed to setup cgroup: {}".format(str(e))) diff --git a/scripts/usb_scan.py b/scripts/usb_scan.py index 3593796d9e8..8cb25e79662 100755 --- a/scripts/usb_scan.py +++ b/scripts/usb_scan.py @@ -21,6 +21,7 @@ # 2. check if device can be passed through based on policy file # 3. return the device info to XAPI in json format +# pytye: disable import abc import argparse @@ -32,6 +33,7 @@ import pyudev import xcp.logger as log +from typing import Any def log_list(l): for s in l: @@ -56,12 +58,13 @@ def hex_equal(h1, h2): return False -class UsbObject(dict): +class UsbObject(dict[str, str]): """ Base class of USB classes, save USB properties in dict node(str): the key, device node """ - __metaclass__ = abc.ABCMeta + if sys.version_info < (3, 0): + __metaclass__ = abc.ABCMeta def __init__(self, node): super(UsbObject, self).__init__() @@ -220,7 +223,7 @@ def add_interface(self, interface): :return: None """ if interface in self.interfaces: - log.debug("overriding existing interface: " + interface) + log.debug("overriding existing interface: ", interface) self.interfaces.remove(interface) self.interfaces.add(interface) @@ -358,7 +361,7 @@ def __init__(self): Note: hubs are never allowed to pass through """ - self.rule_list = [] + self.rule_list = [] # type: list[dict[str, Any]] try: with open(self._PATH, "r") as f: log.debug("=== policy file begin") @@ -424,7 +427,7 @@ def parse_line(self, line): # 3. parse action # \s*(ALLOW|DENY)\s* - rule = {} + rule = {} # type: dict[str, Any] if action.lower() == "allow": rule[self._ALLOW] = True elif action.lower() == "deny": diff --git a/scripts/yum/__init__.py b/scripts/yum/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/yum/plugins.py b/scripts/yum/plugins.py new file mode 100755 index 00000000000..6c88c3239e1 --- /dev/null +++ b/scripts/yum/plugins.py @@ -0,0 +1,781 @@ +#! /usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2005 Duke University + +import os +import glob +import imp +import warnings +import atexit +import gettext +import logging +import logginglevels +from constants import * +import config +from config import ParsingError, ConfigParser +import Errors +from parser import ConfigPreProcessor + +from textwrap import fill +import fnmatch + +from weakref import proxy as weakref + +from yum import _ + +from yum.i18n import utf8_width + +# TODO: expose rpm package sack objects to plugins (once finished) +# TODO: allow plugins to use the existing config stuff to define options for +# their own configuration files (would replace confString() etc). +# TODO: expose progress bar interface +# TODO "log" slot? To allow plugins to do customised logging/history (say to a +# SQL db) +# TODO: consistent case of YumPlugins methods +# TODO: allow plugins to extend shell commands +# TODO: allow plugins to define new repository types +# TODO: More developer docs: use epydoc as API begins to stablise + + +# The API_VERSION constant defines the current plugin API version. It is used +# to decided whether or not plugins can be loaded. It is compared against the +# 'requires_api_version' attribute of each plugin. The version number has the +# format: "major_version.minor_version". +# +# For a plugin to be loaded the major version required by the plugin must match +# the major version in API_VERSION. Additionally, the minor version in +# API_VERSION must be greater than or equal the minor version required by the +# plugin. +# +# If a change to yum is made that break backwards compatibility wrt the plugin +# API, the major version number must be incremented and the minor version number +# reset to 0. If a change is made that doesn't break backwards compatibility, +# then the minor number must be incremented. +API_VERSION = '2.7' + +class DeprecatedInt(int): + """A simple int subclass that is used to check when a deprecated + constant is used. + """ + +# Plugin types +TYPE_CORE = 0 +TYPE_INTERACTIVE = 1 +TYPE_INTERFACE = DeprecatedInt(1) +ALL_TYPES = (TYPE_CORE, TYPE_INTERACTIVE) + +# Mapping of slots to conduit classes +SLOT_TO_CONDUIT = { + 'config': 'ConfigPluginConduit', + 'postconfig': 'PostConfigPluginConduit', + 'init': 'InitPluginConduit', + 'args': 'ArgsPluginConduit', + 'predownload': 'DownloadPluginConduit', + 'postdownload': 'DownloadPluginConduit', + 'prelistenabledrepos': 'PreRepoSetupPluginConduit', + 'prereposetup': 'PreRepoSetupPluginConduit', + 'postreposetup': 'PostRepoSetupPluginConduit', + 'close': 'PluginConduit', + 'clean': 'PluginConduit', + 'pretrans': 'MainPluginConduit', + 'posttrans': 'MainPluginConduit', + 'preverifytrans': 'MainPluginConduit', + 'postverifytrans': 'MainPluginConduit', + 'exclude': 'MainPluginConduit', + 'preresolve': 'DepsolvePluginConduit', + 'postresolve': 'DepsolvePluginConduit', + 'historybegin': 'HistoryPluginConduit', + 'historyend': 'HistoryPluginConduit', + 'compare_providers': 'CompareProvidersPluginConduit', + 'verify_package': 'VerifyPluginConduit', + } + +# Enumerate all slot names +SLOTS = sorted(SLOT_TO_CONDUIT.keys()) + +class PluginYumExit(Exception): + """Exception that can be raised by plugins to signal that yum should stop.""" + + def __init__(self, value="", translation_domain=""): + self.value = value + self.translation_domain = translation_domain + def __str__(self): + if self.translation_domain: + return gettext.dgettext(self.translation_domain, self.value) + else: + return self.value + +class YumPlugins: + """Manager class for Yum plugins.""" + + def __init__(self, base, searchpath, optparser=None, types=None, + pluginconfpath=None,disabled=None,enabled=None): + '''Initialise the instance. + + @param base: The + @param searchpath: A list of paths to look for plugin modules. + @param optparser: The OptionParser instance for this run (optional). + Use to allow plugins to extend command line options. + @param types: A sequence specifying the types of plugins to load. + This should be sequnce containing one or more of the TYPE_... + constants. If None (the default), all plugins will be loaded. + @param pluginconfpath: A list of paths to look for plugin configuration + files. Defaults to "/etc/yum/pluginconf.d". + ''' + if not pluginconfpath: + pluginconfpath = ['/etc/yum/pluginconf.d'] + + self.searchpath = searchpath + self.pluginconfpath = pluginconfpath + self.base = weakref(base) + self.optparser = optparser + self.cmdline = (None, None) + self.verbose_logger = logging.getLogger("yum.verbose.YumPlugins") + self.disabledPlugins = disabled + self.enabledPlugins = enabled + if types is None: + types = ALL_TYPES + if not isinstance(types, (list, tuple)): + types = (types,) + + if id(TYPE_INTERFACE) in [id(t) for t in types]: + self.verbose_logger.log(logginglevels.INFO_2, + 'Deprecated constant TYPE_INTERFACE during plugin ' + 'initialization.\nPlease use TYPE_INTERACTIVE instead.') + + self._importplugins(types) + + self.cmdlines = {} + + # Call close handlers when yum exit's + if self._pluginfuncs['close']: + self.verbose_logger.error( + _('One or more plugins uses "close" handling but should use atexit directly.')) + atexit.register(self.run, 'close') + + # Let plugins register custom config file options + self.run('config') + + def run(self, slotname, **kwargs): + """Run all plugin functions for the given slot. + + :param slotname: a string representing the name of the slot to + run the plugins for + :param kwargs: keyword arguments that will be simply passed on + to the plugins + """ + # Determine handler class to use + conduitcls = SLOT_TO_CONDUIT.get(slotname, None) + if conduitcls is None: + raise ValueError('unknown slot name "%s"' % slotname) + conduitcls = eval(conduitcls) # Convert name to class object + + for modname, func in self._pluginfuncs[slotname]: + self.verbose_logger.log(logginglevels.DEBUG_4, + 'Running "%s" handler for "%s" plugin', + slotname, modname) + + _, conf = self._plugins[modname] + func(conduitcls(self, self.base, conf, **kwargs)) + + def _importplugins(self, types): + '''Load plugins matching the given types. + ''' + + # Initialise plugin dict + self._plugins = {} + self._pluginfuncs = {} + for slot in SLOTS: + self._pluginfuncs[slot] = [] + + # Import plugins + self._used_disable_plugin = set() + self._used_enable_plugin = set() + for dir in self.searchpath: + if not os.path.isdir(dir): + continue + for modulefile in sorted(glob.glob('%s/*.py' % dir)): + self._loadplugin(modulefile, types) + + # If we are in verbose mode we get the full 'Loading "blah" plugin' lines + if (self._plugins and + not self.verbose_logger.isEnabledFor(logginglevels.DEBUG_3)): + # Mostly copied from YumOutput._outKeyValFill() + key = _("Loaded plugins: ") + val = ", ".join(sorted(self._plugins)) + nxt = ' ' * (utf8_width(key) - 2) + ': ' + width = 80 + if hasattr(self.base, 'term'): + width = self.base.term.columns + self.verbose_logger.log(logginglevels.INFO_2, + fill(val, width=width, initial_indent=key, + subsequent_indent=nxt)) + + if self.disabledPlugins: + for wc in self.disabledPlugins: + if wc not in self._used_disable_plugin: + self.verbose_logger.log(logginglevels.INFO_2, + _("No plugin match for: %s") % wc) + del self._used_disable_plugin + if self.enabledPlugins: + for wc in self.enabledPlugins: + if wc not in self._used_enable_plugin: + self.verbose_logger.log(logginglevels.INFO_2, + _("No plugin match for: %s") % wc) + del self._used_enable_plugin + + @staticmethod + def _plugin_cmdline_match(modname, plugins, used): + """ Check if this plugin has been temporary enabled/disabled. """ + if plugins is None: + return False + + for wc in plugins: + if fnmatch.fnmatch(modname, wc): + used.add(wc) + return True + + return False + + + def _loadplugin(self, modulefile, types): + '''Attempt to import a plugin module and register the hook methods it + uses. + ''' + dir, modname = os.path.split(modulefile) + modname = modname.split('.py')[0] + + # This should really work like enable/disable repo. and be based on the + # cmd line order ... but the API doesn't really allow that easily. + # FIXME: Fix for 4.* (lol) + # Do disabled cmd line checks before loading the module code. + if (self._plugin_cmdline_match(modname, self.disabledPlugins, + self._used_disable_plugin) and + not self._plugin_cmdline_match(modname, self.enabledPlugins, + self._used_enable_plugin)): + return + + conf = self._getpluginconf(modname) + if (not conf or + (not config.getOption(conf, 'main', 'enabled', + config.BoolOption(False)) and + not self._plugin_cmdline_match(modname, self.enabledPlugins, + self._used_enable_plugin))): + self.verbose_logger.debug(_('Not loading "%s" plugin, as it is disabled'), modname) + return + + try: + fp, pathname, description = imp.find_module(modname, [dir]) + try: + module = imp.load_module(modname, fp, pathname, description) + finally: + fp.close() + except: + if self.verbose_logger.isEnabledFor(logginglevels.DEBUG_4): + raise # Give full backtrace: + self.verbose_logger.error(_('Plugin "%s" can\'t be imported') % + modname) + return + + # Check API version required by the plugin + if not hasattr(module, 'requires_api_version'): + self.verbose_logger.error( + _('Plugin "%s" doesn\'t specify required API version') % + modname) + return + if not apiverok(API_VERSION, module.requires_api_version): + self.verbose_logger.error( + _('Plugin "%s" requires API %s. Supported API is %s.') % ( + modname, + module.requires_api_version, + API_VERSION, + )) + return + + # Check plugin type against filter + plugintypes = getattr(module, 'plugin_type', ALL_TYPES) + if not isinstance(plugintypes, (list, tuple)): + plugintypes = (plugintypes,) + + if len(plugintypes) < 1: + return + for plugintype in plugintypes: + if id(plugintype) == id(TYPE_INTERFACE): + self.verbose_logger.log(logginglevels.INFO_2, + 'Plugin "%s" uses deprecated constant ' + 'TYPE_INTERFACE.\nPlease use TYPE_INTERACTIVE ' + 'instead.', modname) + + if plugintype not in types: + return + + self.verbose_logger.log(logginglevels.DEBUG_3, _('Loading "%s" plugin'), + modname) + + # Store the plugin module and its configuration file + if modname not in self._plugins: + self._plugins[modname] = (module, conf) + else: + raise Errors.ConfigError(_('Two or more plugins with the name "%s" ' \ + 'exist in the plugin search path') % modname) + + for slot in SLOTS: + funcname = slot+'_hook' + if hasattr(module, funcname): + self._pluginfuncs[slot].append( + (modname, getattr(module, funcname)) + ) + + def _getpluginconf(self, modname): + '''Parse the plugin specific configuration file and return a + IncludingConfigParser instance representing it. Returns None if there + was an error reading or parsing the configuration file. + ''' + for dir in self.pluginconfpath: + conffilename = os.path.join(dir, modname + ".conf") + if os.access(conffilename, os.R_OK): + # Found configuration file + break + self.verbose_logger.log(logginglevels.INFO_2, _("Configuration file %s not found") % conffilename) + else: # for + # Configuration files for the plugin not found + self.verbose_logger.log(logginglevels.INFO_2, _("Unable to find configuration file for plugin %s") + % modname) + return None + parser = ConfigParser() + confpp_obj = ConfigPreProcessor(conffilename) + try: + parser.readfp(confpp_obj) + except ParsingError as e: + raise Errors.ConfigError("Couldn't parse %s: %s" % (conffilename, + str(e))) + return parser + + def setCmdLine(self, opts, commands): + """Set the parsed command line options so that plugins can + access them. + + :param opts: a dictionary containing the values of the command + line options + :param commands: a list of command line arguments passed to yum + """ + self.cmdline = (opts, commands) + + +class DummyYumPlugins: + """This class provides basic emulation of the :class:`YumPlugins` + class. It exists so that calls to plugins.run() don't fail if + plugins aren't in use. + """ + def run(self, *args, **kwargs): + """Do nothing. All arguments are unused.""" + + pass + + def setCmdLine(self, *args, **kwargs): + """Do nothing. All arguments are unused.""" + + pass + +class PluginConduit: + """A conduit class to transfer information between yum and the + plugin. + """ + def __init__(self, parent, base, conf): + self._parent = parent + self._base = base + self._conf = conf + + self.logger = logging.getLogger("yum.plugin") + self.verbose_logger = logging.getLogger("yum.verbose.plugin") + + def info(self, level, msg): + """Send an info message to the logger. + + :param level: the level of the message to send + :param msg: the message to send + """ + converted_level = logginglevels.logLevelFromDebugLevel(level) + self.verbose_logger.log(converted_level, msg) + + def error(self, level, msg): + """Send an error message to the logger. + + :param level: the level of the message to send + :param msg: the message to send + """ + converted_level = logginglevels.logLevelFromErrorLevel(level) + self.logger.log(converted_level, msg) + + def promptYN(self, msg, prompt=None): + """Return a yes or no response, either from assumeyes already + being set, or from prompting the user. + + :param msg: the message to show to the user + :param prompt: the question to ask the user (optional); defaults to 'Is this ok [y/N]: ' + :return: 1 if the response is yes, and 0 if the response is no + """ + self.info(2, msg) + if self._base.conf.assumeno: + return False + if self._base.conf.assumeyes: + return True + else: + kwargs = {'prompt': prompt} if prompt else {} + return bool(self._base.userconfirm(**kwargs)) + + def getYumVersion(self): + """Return a string representing the current version of yum.""" + + import yum + return yum.__version__ # pytype: ignore=module-attr + + def getOptParser(self): + """Return the :class:`optparse.OptionParser` instance for this + execution of Yum. In the "config" and "init" slots a plugin + may add extra options to this instance to extend the command + line options that Yum exposes. In all other slots a plugin + may only read the :class:`OptionParser` instance. Any + modification of the instance at this point will have no + effect. See the + :func:`PreRepoSetupPluginConduit.getCmdLine` method for + details on how to retrieve the parsed values of command line + options. + + :return: the global :class:`optparse.OptionParser` instance used by + Yum. May be None if an OptionParser isn't in use + """ + # ' xemacs highlighting hack + # This isn't API compatible :( + # return self._parent.optparser.plugin_option_group + return self._parent.optparser + + def confString(self, section, opt, default=None): + """Read a string value from the plugin's own configuration file. + + :param section: configuration file section to read + :param opt: option name to read + :param default: value to read if the option is missing + :return: string option value read, or default if option was missing + """ + # ' xemacs highlighting hack + return config.getOption(self._conf, section, opt, config.Option(default)) + + def confInt(self, section, opt, default=None): + """Read an integer value from the plugin's own configuration file. + + :param section: configuration file section to read + :param opt: option name to read + :param default: value to read if the option is missing + + :return: the integer option value read, or *default* if the + option was missing or could not be parsed + """ + return config.getOption(self._conf, section, opt, config.IntOption(default)) + + def confFloat(self, section, opt, default=None): + """Read a float value from the plugin's own configuration file. + + :param section: configuration file section to read + :param opt: option name to read + :param default: value to read if the option is missing + :return: float option value read, or *default* if the option was + missing or could not be parsed + """ + return config.getOption(self._conf, section, opt, config.FloatOption(default)) + + def confBool(self, section, opt, default=None): + """Read a boolean value from the plugin's own configuration file + + :param section: configuration file section to read + :param opt: option name to read + :param default: value to read if the option is missing + :return: boolean option value read, or *default* if the option + was missing or could not be parsed + """ + return config.getOption(self._conf, section, opt, config.BoolOption(default)) + + def confList(self, section, opt, default=None): + """Read a boolean value from the plugin's own configuration file + + :param section: configuration file section to read + :param opt: option name to read + :param default: value to read if the option is missing + :return: boolean option value read, or *default* if the option + was missing or could not be parsed + """ + return config.getOption(self._conf, section, opt, config.ListOption(default)) + + def registerPackageName(self, name): + """Register the name of a package to use. + + :param name: the name of the package to register + """ + self._base.run_with_package_names.add(name) + + +class ConfigPluginConduit(PluginConduit): + """A conduit for use in the config slot.""" + + def registerOpt(self, name, valuetype, where, default): + """Deprecated. Register a yum configuration file option. + + :param name: name of the new option + :param valuetype: option type (PLUG_OPT_BOOL, PLUG_OPT_STRING, etc.) + :param where: where the option should be available in the config file + (PLUG_OPT_WHERE_MAIN, PLUG_OPT_WHERE_REPO, etc) + :param default: default value for the option if it is not set by the user + """ + warnings.warn('registerOpt() will go away in a future version of Yum.\n' + 'Please manipulate config.YumConf and config.RepoConf directly.', + DeprecationWarning) + + type2opt = { + PLUG_OPT_STRING: config.Option, + PLUG_OPT_INT: config.IntOption, + PLUG_OPT_BOOL: config.BoolOption, + PLUG_OPT_FLOAT: config.FloatOption, + } + + if where == PLUG_OPT_WHERE_MAIN: + setattr(config.YumConf, name, type2opt[valuetype](default)) + + elif where == PLUG_OPT_WHERE_REPO: + setattr(config.RepoConf, name, type2opt[valuetype](default)) + + elif where == PLUG_OPT_WHERE_ALL: + option = type2opt[valuetype](default) + setattr(config.YumConf, name, option) + setattr(config.RepoConf, name, config.Inherit(option)) + + def registerCommand(self, command): + """Register a new command. + + :param command: the command to register + :raises: :class:`yum.Errors.ConfigError` if the registration + of commands is not supported + """ + if hasattr(self._base, 'registerCommand'): + self._base.registerCommand(command) + else: + raise Errors.ConfigError(_('registration of commands not supported')) + +class PostConfigPluginConduit(ConfigPluginConduit): + """Conduit for use in the postconfig slot.""" + + def getConf(self): + """Return a dictionary containing the values of the + configuration options. + + :return: a dictionary containing the values of the + configuration options + """ + return self._base.conf + +class InitPluginConduit(PluginConduit): + """Conduit for use in the init slot.""" + + def getConf(self): + """Return a dictionary containing the values of the + configuration options. + + :return: a dictionary containing the values of the + configuration options + """ + return self._base.conf + + def getRepos(self): + """Return Yum's container object for all configured repositories. + + :return: Yum's :class:`yum.repos.RepoStorage` instance + """ + return self._base.repos + +class ArgsPluginConduit(InitPluginConduit): + """Conduit for dealing with command line arguments.""" + + def __init__(self, parent, base, conf, args): + InitPluginConduit.__init__(self, parent, base, conf) + self._args = args + + def getArgs(self): + """Return a list of the command line arguments passed to yum. + + :return: a list of the command line arguments passed to yum + """ + return self._args + +class PreRepoSetupPluginConduit(InitPluginConduit): + """Conduit for use in the prererosetup slot.""" + + + def getCmdLine(self): + """Return parsed command line options. + + :return: (options, commands) as returned by :class:`OptionParser.parse_args()` + """ + return self._parent.cmdline + + def getRpmDB(self): + """Return a representation of the local RPM database. This + allows querying of installed packages. + + :return: a :class:`yum.rpmUtils.RpmDBHolder` instance + """ + return self._base.rpmdb + +class PostRepoSetupPluginConduit(PreRepoSetupPluginConduit): + """Conduit for use in the postreposetup slot.""" + + def getGroups(self): + """Return group information. + + :return: :class:`yum.comps.Comps` instance + """ + return self._base.comps + +class DownloadPluginConduit(PostRepoSetupPluginConduit): + """Conduit for use in the download slots.""" + + def __init__(self, parent, base, conf, pkglist, errors=None): + PostRepoSetupPluginConduit.__init__(self, parent, base, conf) + self._pkglist = pkglist + self._errors = errors + + def getDownloadPackages(self): + """Return a list of package objects representing packages to be + downloaded. + + :return: a list of package object representing packages to be + downloaded + """ + return self._pkglist + + def getErrors(self): + """Return a dictionary of download errors. + + :return: a dictionary of download errors. This dictionary is + indexed by package object. Each element is a list of + strings describing the error + """ + if not self._errors: + return {} + return self._errors + +class MainPluginConduit(PostRepoSetupPluginConduit): + """Main conduit class for plugins. Many other conduit classes + will inherit from this class. + """ + def getPackages(self, repo=None): + """Return a list of packages. + + :param repo: the repo to return a packages from + :return: a list of package objects + """ + if repo: + arg = repo.id + else: + arg = None + return self._base.pkgSack.returnPackages(arg) + + def getPackageByNevra(self, nevra): + """Retrieve a package object from the packages loaded by Yum using + nevra information. + + :param nevra: a tuple holding (name, epoch, version, release, arch) + for a package + :return: a :class:`yum.packages.PackageObject` instance (or subclass) + """ + return self._base.getPackageObject(nevra) + + def delPackage(self, po): + """Delete the given package from the package sack. + + :param po: the package object to delete + """ + po.repo.sack.delPackage(po) + + def getTsInfo(self): + """Return transaction set. + + :return: the transaction set + """ + return self._base.tsInfo + +class DepsolvePluginConduit(MainPluginConduit): + """Conduit for use in solving dependencies.""" + + def __init__(self, parent, base, conf, rescode=None, restring=[]): + MainPluginConduit.__init__(self, parent, base, conf) + self.resultcode = rescode + self.resultstring = restring + + @property + def missing_requires(self): + """Boolean indicating if depsolving failed due to missing dependencies.""" + return self._base._missing_requires + + def pretty_output_restring(self): + return '\n'.join(prefix % msg for prefix, msg in self._base.pretty_output_restring(self.resultstring)) + +class CompareProvidersPluginConduit(MainPluginConduit): + """Conduit to compare different providers of packages.""" + + def __init__(self, parent, base, conf, providers_dict={}, reqpo=None): + MainPluginConduit.__init__(self, parent, base, conf) + self.packages = providers_dict + self.reqpo = reqpo + +class HistoryPluginConduit(MainPluginConduit): + """Conduit to access information about the yum history.""" + + def __init__(self, parent, base, conf, rescode=None, restring=[]): + MainPluginConduit.__init__(self, parent, base, conf) + self.history = self._base.history + +class VerifyPluginConduit(MainPluginConduit): + """Conduit to verify packages.""" + + def __init__(self, parent, base, conf, verify_package): + MainPluginConduit.__init__(self, parent, base, conf) + self.verify_package = verify_package + +def parsever(apiver): + """Parse a string representing an api version. + + :param apiver: a string representing an api version + :return: a tuple containing the major and minor version numbers + """ + maj, min = apiver.split('.') + return int(maj), int(min) + +def apiverok(a, b): + """Return true if API version "a" supports API version "b" + + :param a: a string representing an api version + :param b: a string representing an api version + + :return: whether version *a* supports version *b* + """ + a = parsever(a) + b = parsever(b) + + if a[0] != b[0]: + return 0 + + if a[1] >= b[1]: + return 1 + + return 0 diff --git a/xcp/__init__.py b/xcp/__init__.py new file mode 100644 index 00000000000..7320295a872 --- /dev/null +++ b/xcp/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2013, Citrix Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from . import compat +from . import logger \ No newline at end of file diff --git a/xcp/cmd.py b/xcp/cmd.py new file mode 100644 index 00000000000..394f0b0e86c --- /dev/null +++ b/xcp/cmd.py @@ -0,0 +1,135 @@ +# Copyright (c) 2013, Citrix Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Command processing""" + +import subprocess +import sys +from typing import Any, cast + +from xcp import logger +from xcp.compat import open_defaults_for_utf8_text + + +def _encode_command_to_bytes(command): + # When the locale not an UTF-8 locale, Python3.6 Popen can't deal with ord() >= 128 + # when the command contains strings, not bytes. Therefore, convert any strings to bytes: + if sys.version_info >= (3, 0) and not isinstance(command, bytes): + if isinstance(command, str): # Encode str because Python 3.6 uses fsencode("ascii") + return command.encode() # if it has been started without an UTF-8 locale set. + if not hasattr(command, "__iter__") and not hasattr(command, "__getitem__"): + raise TypeError("command must be str, bytes or an iterable/sequence") + command = list(command) # Get a copy of the iterable or sequence as list + for idx, arg in enumerate(command): + if isinstance(arg, str): # and encode() any strings in it to bytes, because + command[idx] = arg.encode() # Python 3.6 could fail in fsencode("ascii") + return command + +def runCmd(command, with_stdout=False, with_stderr=False, inputtext=None, **kwargs): + # type: (bytes | str | list[str], bool, bool, bytes | str | None, Any) -> Any + # sourcery skip: assign-if-exp, hoist-repeated-if-condition, reintroduce-else + + if inputtext is not None: + kwargs["mode"] = "t" if isinstance(inputtext, str) else "b" + if with_stdout or with_stderr: + open_defaults_for_utf8_text(None, kwargs) + kwargs.pop("mode", "") + + command = _encode_command_to_bytes(command) + + # bufsize=1 means buffered in 2.7, but means line buffered in Py3 (not valid in binary mode) + # bufsize=-1 is the equivalent of bufsize=1 in Python >= 3.3.1 + + # pylint: disable-next=unexpected-keyword-arg + cmd = subprocess.Popen(command, bufsize=(1 if sys.version_info < (3, 3) else -1), + stdin=cast(int, inputtext and subprocess.PIPE or None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=not isinstance(command, list), + **kwargs) + (out, err) = cmd.communicate(cast(str, inputtext)) + rv = cmd.returncode + + l = "ran %s; rc %d" % (str(command), rv) + if inputtext and isinstance(inputtext, str): + l += " with input %s" % inputtext + if out != "" and isinstance(out, str): + l += "\nSTANDARD OUT:\n" + out + if err != "" and isinstance(err, str): + l += "\nSTANDARD ERROR:\n" + err + + for line in l.split('\n'): + logger.debug(line) + + if with_stdout and with_stderr: + return rv, out, err + if with_stdout: + return rv, out + if with_stderr: + return rv, err + return rv + +class OutputCache(object): + def __init__(self): + self.cache = {} + + def fileContents(self, fn, *args, **kwargs): + mode, other_kwargs = open_defaults_for_utf8_text(args, kwargs) + key = "file:" + fn + "," + mode + (str(other_kwargs) if other_kwargs else "") + if key not in self.cache: + logger.debug("Opening " + key) + # pylint: disable=unspecified-encoding + with open(fn, *args, **kwargs) as f: + self.cache[key] = f.read() if "b" in mode else "".join(f.readlines()) + return self.cache[key] + + def runCmd(self, command, with_stdout=False, with_stderr=False, inputtext=None, **kwargs): + key = str(command) + str(kwargs.get("mode")) + str(inputtext) + rckey = 'cmd.rc:' + key + outkey = 'cmd.out:' + key + errkey = 'cmd.err:' + key + cache = self.cache + if with_stdout and with_stderr: + if rckey not in cache: + cache[rckey], cache[outkey], cache[errkey] = runCmd( # pyright: ignore + command, True, True, inputtext, **kwargs + ) + return self.cache[rckey], self.cache[outkey], self.cache[errkey] + if with_stdout: + if rckey not in cache: + cache[rckey], cache[outkey] = runCmd( # pyright: ignore + command, True, False, inputtext, **kwargs + ) + return self.cache[rckey], self.cache[outkey] + if with_stderr: + if rckey not in cache: + cache[rckey], cache[errkey] = runCmd( # pyright: ignore + command, False, True, inputtext, **kwargs + ) + return self.cache[rckey], self.cache[errkey] + if rckey not in cache: + cache[rckey] = runCmd(command, False, False, inputtext, **kwargs) + return self.cache[rckey] + + def clearCache(self): + self.cache.clear() diff --git a/xcp/compat.py b/xcp/compat.py new file mode 100644 index 00000000000..9a08772e1ed --- /dev/null +++ b/xcp/compat.py @@ -0,0 +1,59 @@ +"""Helper module for setting up binary or UTF-8 I/O for Popen and open in Python 3.6 and newer""" +# pyright: strict +# pyright: reportTypeCommentUsage=false +# pyright: reportUnknownParameterType=false +# pytype: disable=bad-return-type,ignored-type-comment,invalid-function-type-comment +# See README-Unicode.md for Details +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import IO, Any # pylint: disable=unused-import # pragma: no cover + +def open_textfile(filename, mode, encoding="utf-8", **kwargs): + # type:(str, str, str, Any) -> IO[str] + """For best type checking and type inference, wrap open_with_codec_handling returning IO[str]""" + if "b" in mode: + raise ValueError("open_textfile returns IO[str]: mode must not contain 'b'") + return open_with_codec_handling(filename, mode, encoding, **kwargs) + +# pylint: disable=unspecified-encoding +if sys.version_info >= (3, 0): + open_utf8 = {"encoding": "utf-8", "errors": "replace"} + + def open_with_codec_handling(filename, mode="r", encoding="utf-8", **kwargs): + # type:(str, str, str, Any) -> IO[Any] + """Helper for open(): Handle UTF-8: Default to encoding="utf-8", errors="replace" for Py3""" + if "b" in mode: + # Binary mode: just call open() unmodified: + return open(filename, mode, **kwargs) # pragma: no cover + # Text mode: default to UTF-8 with error handling to replace malformed UTF-8 sequences + # Needed for Python 3.6 when no UTF-8 locale is set: + kwargs.setdefault("encoding", encoding) + kwargs.setdefault("errors", "replace") # Simple codec error handler: Replace malformed char + return open(filename, mode, **kwargs) # type: ignore[call-overload] + +else: + open_utf8 = {} + + def open_with_codec_handling(filename, mode="r", encoding="", errors="", **kwargs): + # type:(str, str, str, str, str) -> IO[Any] + """open() wrapper to pass mode and **kwargs to open(), ignores endcoding and errors args""" + _ = encoding + _ = errors + return open(filename, mode, **kwargs) + + +def open_defaults_for_utf8_text(args, kwargs): + # type:(tuple[Any, ...] | None, Any) -> tuple[str, Any] + """Setup keyword arguments for UTF-8 text mode with codec error handler to replace chars""" + other_kwargs = kwargs.copy() + mode = other_kwargs.pop("mode", "") + if args: + mode = args[0] + if sys.version_info < (3, 0): + return mode, other_kwargs + if sys.version_info >= (3, 0) and "b" not in mode: + kwargs.setdefault("encoding", "utf-8") + kwargs.setdefault("errors", "replace") + return mode or "r", other_kwargs diff --git a/xcp/logger.py b/xcp/logger.py new file mode 100644 index 00000000000..19ed6de6e4c --- /dev/null +++ b/xcp/logger.py @@ -0,0 +1,150 @@ +# Copyright (c) 2013, Citrix Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Logging support with backwards compatibility for xelogging""" + +import fcntl +import os +import os.path +import sys +import traceback +import logging +import logging.handlers +from typing import TYPE_CHECKING, TextIO + +from .compat import open_with_codec_handling + +if TYPE_CHECKING: + from typing import IO, Any, Union + from logging import StreamHandler + from logging.handlers import RotatingFileHandler + LoggingStreamHandler = Union[RotatingFileHandler, StreamHandler[IO[Any]], StreamHandler[TextIO]] + +LOG = logging.getLogger() +LOG.setLevel(logging.NOTSET) +FORMAT = logging.Formatter( + "%(levelname)- 9.9s[%(asctime)s] %(message)s", + "%F %T") + +our_handlers = [] # type: list[logging.Handler] + +def openLog(lfile, level=logging.INFO): + # type:(Union[str, TextIO], int) -> bool + """Add a new file target to be logged to""" + + try: + # if lfile is a string, assume we need to open() it + if isinstance(lfile, str): + h = open_with_codec_handling(lfile, "a") + if h.isatty(): + # pytype: disable=wrong-arg-types # False positive in pytype-2023.07.21 + handler = logging.StreamHandler(h) # type: LoggingStreamHandler + # pytype: enable=wrong-arg-types + else: + h.close() + handler = logging.handlers.RotatingFileHandler(lfile, + maxBytes=2**31) + old = fcntl.fcntl(handler.stream.fileno(), fcntl.F_GETFD) + fcntl.fcntl(handler.stream.fileno(), + fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) + + # or if it is not a string, assume its a file-like object + else: + handler = logging.StreamHandler(lfile) + + except Exception: + if len(LOG.handlers): + log("Error opening %s as a log output." % str(lfile)) + else: + sys.stderr.write("Error opening %s as a log output." % str(lfile)) + return False + + handler.setFormatter(FORMAT) + handler.setLevel(level) + LOG.addHandler(handler) + our_handlers.append(handler) + return True + +def closeLogs(): + """Close all logs""" + handlers_to_remove = list(our_handlers) + for h in handlers_to_remove: + our_handlers.remove(h) + LOG.removeHandler(h) + h.close() + +def logToStdout(level=logging.INFO): + """Log to stdout""" + return openLog(sys.stdout, level) + +def logToStderr(level=logging.INFO): + """Log to stderr""" + return openLog(sys.stderr, level) + +def logToSyslog(ident = os.path.basename(sys.argv[0]), level = logging.INFO, + facility = logging.handlers.SysLogHandler.LOG_USER): + """Log to syslog""" + if os.path.exists("/dev/log"): + syslog = logging.handlers.SysLogHandler("/dev/log", facility) + else: + syslog = logging.handlers.SysLogHandler( + ('localhost', logging.handlers.SYSLOG_UDP_PORT), facility) + syslog.setLevel(level) + fmt = logging.Formatter(ident+" %(levelname)s: %(message)s") + syslog.setFormatter(fmt) + LOG.addHandler(syslog) + +def log(txt): + """ Write txt to the log(s) """ + LOG.info(txt) + +def logException(e): + """ Formats exception and logs it """ + ex = sys.exc_info() + err = traceback.format_exception(*ex) + errmsg = "\n".join([ str(x) for x in e.args ]) + + LOG.critical(errmsg) + LOG.critical(err) + +# export the standard logging calls at the module level + +def debug(*al, **ad): + """debug""" + LOG.debug(*al, **ad) + +def info(*al, **ad): + """info""" + LOG.info(*al, **ad) + +def warning(*al, **ad): + """warning""" + LOG.warning(*al, **ad) + +def error(*al, **ad): + """error""" + LOG.error(*al, **ad) + +def critical(*al, **ad): + """critical""" + LOG.critical(*al, **ad) diff --git a/xcp/logger.pyi b/xcp/logger.pyi new file mode 100644 index 00000000000..06640211341 --- /dev/null +++ b/xcp/logger.pyi @@ -0,0 +1,15 @@ +# manually derived from mypy'sy stubgen tool: +from typing import TextIO, Union + +def openLog(lfile: Union[str, TextIO], level: int = ...) -> bool: ... +def closeLogs() -> None: ... +def logToStdout(level=...): ... +def logToStderr(level=...): ... +def logToSyslog(ident=..., level=..., facility=...) -> None: ... +def log(txt) -> None: ... +def logException(e) -> None: ... +def debug(*al, **ad) -> None: ... +def info(*al, **ad) -> None: ... +def warning(*al, **ad) -> None: ... +def error(*al, **ad) -> None: ... +def critical(*al, **ad) -> None: ...