From ee28a87e1270bc1536daff55849f53c6b14e0a7c Mon Sep 17 00:00:00 2001 From: David Butenhof Date: Thu, 11 Mar 2021 09:36:40 -0500 Subject: [PATCH] Build state tracker (#2074) Define a SQLAlchemy DB class "Dataset" which defines persistent state for a Pbench dataset including most importantly the dataset's username (which is not otherwise recorded until we index) and the dataset's current state so that we can track the progress of a dataset through the Pbench server pipeline. We also support Metadata associated with each Dataset, describing additional information about datasets beyond the "state". For example, the backup component will mark a dataset as "ARCHIVED" and pbench-reindex marks the selected datasets to "REINDEX". A new dataset starts in UPLOADING state and will progress through the steps as we perform operations on it. We're defining both "in progress" -ing steps such as INDEXING as well as completion steps (which are ready for the next operation) such as INDEXED. There are also two "terminal" states, EXPIRED and QUARANTINED, from which a dataset cannot exit. --- .gitignore | 1 + jenkins/development.Dockerfile | 1 + lib/pbench/cli/server/shell.py | 19 +- lib/pbench/server/api/__init__.py | 1 - lib/pbench/server/api/resources/__init__.py | 0 lib/pbench/server/api/resources/upload_api.py | 23 + lib/pbench/server/api/resources/users_api.py | 12 +- lib/pbench/server/database/database.py | 24 +- lib/pbench/server/database/models/tracker.py | 734 ++++++++++++++++++ lib/pbench/server/indexing_tarballs.py | 46 +- lib/pbench/server/utils.py | 9 + lib/pbench/test/unit/server/conftest.py | 2 +- lib/pbench/test/unit/server/test_requests.py | 13 +- .../test/unit/server/test_state_tracker.py | 258 ++++++ lib/pbench/test/unit/server/test_user_auth.py | 2 +- server/bin/gold/test-11.txt | 7 + server/bin/gold/test-16.txt | 9 + server/bin/gold/test-17.txt | 9 + server/bin/gold/test-29.1.txt | 4 + server/bin/gold/test-29.2.txt | 11 + server/bin/gold/test-29.3.txt | 4 + server/bin/gold/test-5.2.txt | 19 + server/bin/gold/test-6.10.txt | 7 + server/bin/gold/test-6.11.txt | 7 + server/bin/gold/test-6.12.txt | 7 + server/bin/gold/test-6.13.txt | 8 + server/bin/gold/test-6.14.txt | 8 + server/bin/gold/test-6.15.txt | 7 + server/bin/gold/test-6.16.txt | 8 + server/bin/gold/test-6.17.txt | 8 + server/bin/gold/test-6.4.txt | 8 + server/bin/gold/test-6.5.txt | 8 + server/bin/gold/test-6.6.txt | 8 + server/bin/gold/test-6.7.txt | 8 + server/bin/gold/test-6.9.txt | 7 + server/bin/gold/test-7.1.txt | 7 + server/bin/gold/test-7.10.txt | 7 + server/bin/gold/test-7.11.txt | 7 + server/bin/gold/test-7.12.txt | 7 + server/bin/gold/test-7.13.txt | 7 + server/bin/gold/test-7.14.txt | 7 + server/bin/gold/test-7.15.txt | 7 + server/bin/gold/test-7.16.txt | 7 + server/bin/gold/test-7.17.txt | 7 + server/bin/gold/test-7.18.txt | 8 + server/bin/gold/test-7.19.txt | 7 + server/bin/gold/test-7.2.0.txt | 7 + server/bin/gold/test-7.2.1.txt | 7 + server/bin/gold/test-7.20.txt | 7 + server/bin/gold/test-7.21.txt | 8 + server/bin/gold/test-7.22.txt | 7 + server/bin/gold/test-7.23.txt | 7 + server/bin/gold/test-7.24.txt | 8 + server/bin/gold/test-7.25.txt | 7 + server/bin/gold/test-7.26.txt | 7 + server/bin/gold/test-7.3.txt | 7 + server/bin/gold/test-7.4.txt | 7 + server/bin/gold/test-7.5.txt | 7 + server/bin/gold/test-7.6.txt | 7 + server/bin/gold/test-7.7.txt | 7 + server/bin/gold/test-7.8.txt | 7 + server/bin/gold/test-7.9.txt | 7 + server/bin/pbench-backup-tarballs.py | 21 + server/bin/pbench-base.sh | 8 + server/bin/pbench-index.py | 4 + server/bin/pbench-reindex.py | 18 + server/bin/pbench-server-prep-shim-002.py | 42 + server/bin/pbench-state-manager | 1 + server/bin/pbench-state-manager.py | 124 +++ server/bin/pbench-sync-satellite.sh | 7 + server/bin/pbench-unpack-tarballs.sh | 25 + server/bin/state/test-16.db | 3 + server/bin/state/test-17.db | 3 + server/bin/state/test-29.2.db | 1 + server/bin/state/test-6.10.db | 1 + server/bin/state/test-6.11.db | 1 + server/bin/state/test-6.12.db | 1 + server/bin/state/test-6.13.db | 2 + server/bin/state/test-6.14.db | 2 + server/bin/state/test-6.15.db | 1 + server/bin/state/test-6.16.db | 2 + server/bin/state/test-6.17.db | 2 + server/bin/state/test-6.4.db | 2 + server/bin/state/test-6.5.db | 2 + server/bin/state/test-6.6.db | 2 + server/bin/state/test-6.7.db | 2 + server/bin/state/test-6.9.db | 1 + server/bin/state/test-7.1.db | 1 + server/bin/state/test-7.10.db | 1 + server/bin/state/test-7.11.db | 1 + server/bin/state/test-7.12.db | 1 + server/bin/state/test-7.13.db | 1 + server/bin/state/test-7.14.db | 1 + server/bin/state/test-7.15.db | 1 + server/bin/state/test-7.16.db | 1 + server/bin/state/test-7.17.db | 1 + server/bin/state/test-7.18.db | 2 + server/bin/state/test-7.19.db | 1 + server/bin/state/test-7.2.0.db | 1 + server/bin/state/test-7.2.1.db | 1 + server/bin/state/test-7.20.db | 1 + server/bin/state/test-7.21.db | 2 + server/bin/state/test-7.22.db | 1 + server/bin/state/test-7.23.db | 1 + server/bin/state/test-7.24.db | 2 + server/bin/state/test-7.25.db | 1 + server/bin/state/test-7.26.db | 1 + server/bin/state/test-7.3.db | 1 + server/bin/state/test-7.4.db | 1 + server/bin/state/test-7.5.db | 1 + server/bin/state/test-7.6.db | 1 + server/bin/state/test-7.7.db | 1 + server/bin/state/test-7.8.db | 1 + server/bin/state/test-7.9.db | 1 + server/bin/unittests | 27 +- server/requirements.txt | 1 + 116 files changed, 1801 insertions(+), 34 deletions(-) create mode 100644 lib/pbench/server/api/resources/__init__.py create mode 100644 lib/pbench/server/database/models/tracker.py create mode 100644 lib/pbench/test/unit/server/test_state_tracker.py create mode 120000 server/bin/pbench-state-manager create mode 100755 server/bin/pbench-state-manager.py create mode 100644 server/bin/state/test-16.db create mode 100644 server/bin/state/test-17.db create mode 100644 server/bin/state/test-29.2.db create mode 100644 server/bin/state/test-6.10.db create mode 100644 server/bin/state/test-6.11.db create mode 100644 server/bin/state/test-6.12.db create mode 100644 server/bin/state/test-6.13.db create mode 100644 server/bin/state/test-6.14.db create mode 100644 server/bin/state/test-6.15.db create mode 100644 server/bin/state/test-6.16.db create mode 100644 server/bin/state/test-6.17.db create mode 100644 server/bin/state/test-6.4.db create mode 100644 server/bin/state/test-6.5.db create mode 100644 server/bin/state/test-6.6.db create mode 100644 server/bin/state/test-6.7.db create mode 100644 server/bin/state/test-6.9.db create mode 100644 server/bin/state/test-7.1.db create mode 100644 server/bin/state/test-7.10.db create mode 100644 server/bin/state/test-7.11.db create mode 100644 server/bin/state/test-7.12.db create mode 100644 server/bin/state/test-7.13.db create mode 100644 server/bin/state/test-7.14.db create mode 100644 server/bin/state/test-7.15.db create mode 100644 server/bin/state/test-7.16.db create mode 100644 server/bin/state/test-7.17.db create mode 100644 server/bin/state/test-7.18.db create mode 100644 server/bin/state/test-7.19.db create mode 100644 server/bin/state/test-7.2.0.db create mode 100644 server/bin/state/test-7.2.1.db create mode 100644 server/bin/state/test-7.20.db create mode 100644 server/bin/state/test-7.21.db create mode 100644 server/bin/state/test-7.22.db create mode 100644 server/bin/state/test-7.23.db create mode 100644 server/bin/state/test-7.24.db create mode 100644 server/bin/state/test-7.25.db create mode 100644 server/bin/state/test-7.26.db create mode 100644 server/bin/state/test-7.3.db create mode 100644 server/bin/state/test-7.4.db create mode 100644 server/bin/state/test-7.5.db create mode 100644 server/bin/state/test-7.6.db create mode 100644 server/bin/state/test-7.7.db create mode 100644 server/bin/state/test-7.8.db create mode 100644 server/bin/state/test-7.9.db diff --git a/.gitignore b/.gitignore index 22ae4bce7b..955260c802 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build dist *~ ????-*.patch +.env .npmrc .yarnrc *.egg-info diff --git a/jenkins/development.Dockerfile b/jenkins/development.Dockerfile index b775c81fe1..ace2d67a0c 100644 --- a/jenkins/development.Dockerfile +++ b/jenkins/development.Dockerfile @@ -97,6 +97,7 @@ RUN \ rsync \ screen \ sos \ + sqlite \ tar \ xz \ && \ diff --git a/lib/pbench/cli/server/shell.py b/lib/pbench/cli/server/shell.py index 42678739cd..c1da7fe417 100644 --- a/lib/pbench/cli/server/shell.py +++ b/lib/pbench/cli/server/shell.py @@ -4,9 +4,12 @@ import sys from configparser import NoSectionError, NoOptionError +from sqlalchemy_utils import database_exists, create_database from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified from pbench.server.api import create_app, get_server_config +from pbench.server.database.database import Database +from pbench.common.logger import get_pbench_logger def app(): @@ -24,12 +27,24 @@ def main(): except (ConfigFileNotSpecified, BadConfig) as e: print(e) sys.exit(1) + logger = get_pbench_logger(__name__, server_config) try: host = str(server_config.get("pbench-server", "bind_host")) port = str(server_config.get("pbench-server", "bind_port")) + db = str(server_config.get("Postgres", "db_uri")) workers = str(server_config.get("pbench-server", "workers")) - except (NoOptionError, NoSectionError) as e: - print(f"{__name__}: ERROR: {e.__traceback__}") + + # Multiple gunicorn workers will attempt to connect to the DB; rather + # than attempt to synchronize them, detect a missing DB (from the + # postgres URI) and create it here. It's safer to do this here, + # where we're single-threaded. + if not database_exists(db): + logger.info("Postgres DB {} doesn't exist", db) + create_database(db) + logger.info("Created DB {}", db) + Database.init_db(server_config, logger) + except (NoOptionError, NoSectionError): + logger.exception(f"{__name__}: ERROR") sys.exit(1) subprocess.run( diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index b6d5026b37..8657c57c96 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -21,7 +21,6 @@ from pbench.server.database.database import Database from pbench.server.api.resources.query_apis.query_month_indices import QueryMonthIndices from pbench.server.api.auth import Auth - from pbench.server.api.resources.users_api import ( RegisterUser, Login, diff --git a/lib/pbench/server/api/resources/__init__.py b/lib/pbench/server/api/resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pbench/server/api/resources/upload_api.py b/lib/pbench/server/api/resources/upload_api.py index d58b8928d0..54a6f10bb0 100644 --- a/lib/pbench/server/api/resources/upload_api.py +++ b/lib/pbench/server/api/resources/upload_api.py @@ -3,10 +3,14 @@ import tempfile import hashlib from pathlib import Path +from http import HTTPStatus + from flask_restful import Resource, abort from flask import request, jsonify from werkzeug.utils import secure_filename from pbench.server.utils import filesize_bytes +from pbench.server.database.models.tracker import Dataset, States + ALLOWED_EXTENSIONS = {"xz"} @@ -109,6 +113,20 @@ def put(self, controller): md5_full_path = Path(path, f"{filename}.md5") bytes_received = 0 + # TODO: Need real user from PUT! + + # Create a tracking dataset object; it'll begin in UPLOADING state + try: + dataset = Dataset(controller=controller, path=tar_full_path, md5=md5sum) + dataset.add() + except Exception: + self.logger.exception("unable to create dataset for {}", filename) + abort( + HTTPStatus.INTERNAL_SERVER_ERROR, message="INTERNAL ERROR", + ) + + self.logger.info("Uploading file {} to {}", filename, dataset) + with tempfile.NamedTemporaryFile(mode="wb", dir=path) as ofp: chunk_size = 4096 self.logger.debug("Writing chunks") @@ -181,6 +199,11 @@ def put(self, controller): ) raise + try: + dataset.advance(States.UPLOADED) + except Exception: + self.logger.exception("Unable to finalize {}", dataset) + abort(HTTPStatus.INTERNAL_SERVER_ERROR, message="INTERNAL ERROR") response = jsonify(dict(message="File successfully uploaded")) response.status_code = 201 return response diff --git a/lib/pbench/server/api/resources/users_api.py b/lib/pbench/server/api/resources/users_api.py index 43c8f4a8ec..cc056bfe7d 100644 --- a/lib/pbench/server/api/resources/users_api.py +++ b/lib/pbench/server/api/resources/users_api.py @@ -92,13 +92,13 @@ def post(self): first_name = user_data.get("first_name") if not first_name: - self.logger.warning("Missing firstName field") - abort(400, message="Missing firstName field") + self.logger.warning("Missing first_name field") + abort(400, message="Missing first_name field") last_name = user_data.get("last_name") if not last_name: - self.logger.warning("Missing lastName field") - abort(400, message="Missing lastName field") + self.logger.warning("Missing last_name field") + abort(400, message="Missing last_name field") try: user = User( @@ -325,8 +325,8 @@ def get(self, username): "message": "Success"/"failure message", "data": { "username": , - "firstName": , - "lastName": , + "first_name": , + "last_name": , "registered_on": registered_on, } } diff --git a/lib/pbench/server/database/database.py b/lib/pbench/server/database/database.py index e8fcf93805..bf20597427 100644 --- a/lib/pbench/server/database/database.py +++ b/lib/pbench/server/database/database.py @@ -7,8 +7,6 @@ class Database: # Create declarative base model that our model can inherit from Base = declarative_base() - # Initialize the db scoped session - db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False)) @staticmethod def get_engine_uri(config, logger): @@ -21,24 +19,20 @@ def get_engine_uri(config, logger): ) return None - # return f"postgresql+{psql_driver}://{psql_username}:{psql_password}@{psql_host}:{psql_port}/{psql_db}" - - @staticmethod - def init_engine(server_config, logger): - try: - return create_engine(Database.get_engine_uri(server_config, logger)) - except Exception: - logger.exception("Exception while creating a sqlalchemy engine") - return None - @staticmethod def init_db(server_config, logger): + # Attach the logger to the base class for models to find + if not hasattr(Database.Base, "logger"): + Database.Base.logger = logger + + # WARNING: # Make sure all the models are imported before this function gets called # so that they will be registered properly on the metadata. Otherwise # metadata will not have any tables and create_all functionality will do nothing - Database.Base.query = Database.db_session.query_property() - engine = create_engine(Database.get_engine_uri(server_config, logger)) Database.Base.metadata.create_all(bind=engine) - Database.db_session.configure(bind=engine) + Database.db_session = scoped_session( + sessionmaker(bind=engine, autocommit=False, autoflush=False) + ) + Database.Base.query = Database.db_session.query_property() diff --git a/lib/pbench/server/database/models/tracker.py b/lib/pbench/server/database/models/tracker.py new file mode 100644 index 0000000000..551c52c4d0 --- /dev/null +++ b/lib/pbench/server/database/models/tracker.py @@ -0,0 +1,734 @@ +import datetime +import enum +from pathlib import Path +import os + +from sqlalchemy import ( + Column, + Integer, + String, + DateTime, + Enum, + ForeignKey, + UniqueConstraint, + event, +) +from sqlalchemy.orm import relationship, validates +from sqlalchemy.exc import SQLAlchemyError + +from pbench.server.database.database import Database + + +class DatasetError(Exception): + """ + DatasetError This is a base class for errors reported by the + Dataset class. It is never raised directly, but + may be used in "except" clauses. + """ + + pass + + +class DatasetSqlError(DatasetError): + """ + DatasetSqlError SQLAlchemy errors reported through Dataset operations. + + The exception will identify the controller and name of the target dataset, + along with the operation being attempted; the __cause__ will specify the + original SQLAlchemy exception. + """ + + def __init__(self, operation, controller, name): + self.operation = operation + self.controller = controller + self.name = name + + def __str__(self): + return f"Error {self.operation} {self.controller}|{self.name}" + + +class DatasetNotFound(DatasetError): + """ + DatasetNotFound Attempt to attach to a Dataset that doesn't exist. + """ + + def __init__(self, controller, name): + self.controller = controller + self.name = name + + def __str__(self): + return f"No dataset {self.controller}|{self.name}" + + +class DatasetBadParameterType(DatasetError): + """ + DatasetBadParameterType A parameter of the wrong type was passed to a + method in the Dataset module. + + The error text will identify the actual value and type, and the expected + type. + """ + + def __init__(self, bad_value, expected_type): + self.bad_value = bad_value + self.expected_type = expected_type + + def __str__(self): + return f'Value "{self.bad_value}" ({type(self.bad_value)}) is not a {type(self.expected_type)}' + + +class DatasetTransitionError(DatasetError): + """ + DatasetTransitionError A base class for errors reporting disallowed + dataset state transitions. It is never raised directly, but + may be used in "except" clauses. + """ + + pass + + +class DatasetTerminalStateViolation(DatasetTransitionError): + """ + DatasetTerminalStateViolation An attempt was made to change the state of + a dataset currently in a terminal state. + + The error text will identify the dataset by controller and name, and both + the current and requested new states. + """ + + def __init__(self, dataset, requested_state): + self.dataset = dataset + self.requested_state = requested_state + + def __str__(self): + return f"Dataset {self.dataset} state {self.dataset.state} is terminal and cannot be advanced to {self.requested_state}" + + +class DatasetBadStateTransition(DatasetTransitionError): + """ + DatasetTransitionError An attempt was made to advance a dataset to a new + state that's not reachable from the current state. + + The error text will identify the dataset by controller and name, and both + the current and requested new states. + """ + + def __init__(self, dataset, requested_state): + self.dataset = dataset + self.requested_state = requested_state + + def __str__(self): + return f"Dataset {self.dataset} desired state {self.requested_state} is not allowed from current state {self.dataset.state}" + + +class MetadataError(DatasetError): + """ + MetadataError A base class for errors reported by the Metadata class. It + is never raised directly, but may be used in "except" clauses. + """ + + def __init__(self, dataset, key): + self.dataset = dataset + self.key = key + + def __str__(self): + return f"Generic error on {self.dataset} key {self.key}" + + +class MetadataSqlError(MetadataError): + """ + MetadataSqlError SQLAlchemy errors reported through Metadata operations. + + The exception will identify the dataset and the metadata key, along with + the operation being attempted; the __cause__ will specify the original + SQLAlchemy exception. + """ + + def __init__(self, operation, dataset, key): + self.operation = operation + super().__init__(dataset, key) + + def __str__(self): + return f"Error {self.operation} {self.dataset} key {self.key}" + + +class MetadataNotFound(MetadataError): + """ + MetadataNotFound An attempt to `get` or remove a Metadata key that isn't + present. + + The error text will identify the dataset and metadata key that was + specified. + """ + + def __init__(self, dataset, key): + super().__init__(dataset, key) + + def __str__(self): + return f"No metadata {self.key} for {self.dataset}" + + +class MetadataKeyError(DatasetError): + """ + MetadataKeyError A base class for metadata key errors in the context of + Metadata errors that have no associated Dataset. It is never + raised directly, but may be used in "except" clauses. + """ + + +class MetadataMissingParameter(MetadataKeyError): + """ + MetadataMissingParameter A Metadata required parameter was not specified. + """ + + def __init__(self, what): + self.what = what + + def __str__(self): + return f"Metadata must specify a {self.what}" + + +class MetadataBadKey(MetadataKeyError): + """ + MetadataBadKey An unsupported metadata key was specified. + + The error text will identify the metadata key that was specified. + """ + + def __init__(self, key): + self.key = key + + def __str__(self): + return f"Metadata key {self.key} is not supported" + + +class MetadataMissingKeyValue(MetadataKeyError): + """ + MetadataMissingKeyValue A value must be specified for the metadata key. + + The error text will identify the metadata key that was specified. + """ + + def __init__(self, key): + self.key = key + + def __str__(self): + return f"Metadata key {self.key} value is required" + + +class MetadataDuplicateKey(MetadataError): + """ + MetadataDuplicateKey An attempt to add a Metadata key that already exists + on the dataset. + + The error text will identify the dataset and metadata key that was + specified. + """ + + def __init__(self, dataset, key): + super().__init__(dataset, key) + + def __str__(self): + return f"{self.dataset} already has metadata key {self.key}" + + +class States(enum.Enum): + """ + States: Track the progress of a dataset (tarball) through the various + stages and operation of the Pbench server. + """ + + UPLOADING = ("Uploading", True) + UPLOADED = ("Uploaded", False) + UNPACKING = ("Unpacking", True) + UNPACKED = ("Unpacked", False) + INDEXING = ("Indexing", True) + INDEXED = ("Indexed", False) + EXPIRING = ("Expiring", True) + EXPIRED = ("Expired", False) + QUARANTINED = ("Quarantined", False) + + def __init__(self, friendly: str, mutating: bool): + """ + __init__ Extended ENUM constructor in order to add a value + to record whether each state is a "busy" state where some + component is mutating the dataset in some way, and a mixed + case "friendly" name for the state. "Mutating" states are + expected to be transient as the component should complete + the mutation, and should usually have "-ing" endings. + + Args: + name (string): Friendly name for the state + mutating (boolean): True if a component is mutating dataset + """ + self.friendly = friendly + self.mutating = mutating + + def __str__(self): + """ + Return the state's friendly name + """ + return self.friendly + + +class Dataset(Database.Base): + """ + Identify a Pbench dataset (tarball plus metadata) + + Columns: + id Generated unique ID of table row + user User associated with the dataset + controller Name of controller node + name Base name of dataset (tarball) + md5 The dataset MD5 hash (Elasticsearch ID) + created Date the dataset was PUT to server + state The current state of the dataset + transition The timestamp of the last state transition + """ + + __tablename__ = "dataset" + + transitions = { + States.UPLOADING: [States.UPLOADED, States.QUARANTINED], + States.UPLOADED: [States.UNPACKING, States.QUARANTINED], + States.UNPACKING: [States.UNPACKED, States.QUARANTINED], + States.UNPACKED: [States.INDEXING, States.QUARANTINED], + States.INDEXING: [States.INDEXED, States.QUARANTINED], + States.INDEXED: [States.INDEXING, States.EXPIRING, States.QUARANTINED], + States.EXPIRING: [States.EXPIRED, States.INDEXED, States.QUARANTINED], + # NOTE: EXPIRED and QUARANTINED do not appear as keys in the table + # because they're terminal states that cannot be exited. + } + + ANTI_USER = "public" # The "user" that owns unowned datasets + + id = Column(Integer, primary_key=True, autoincrement=True) + user = Column(String(255), unique=False, nullable=False, default=ANTI_USER) + controller = Column(String(255), unique=False, nullable=False) + name = Column(String(255), unique=False, nullable=False) + + # FIXME: + # Ideally, `md5` would not be `nullable`: this allows the + # pbench-server-prep-shim-002 utility to construct a Dataset object + # before accessing and checking the MD5 (in order to ensure that we + # always have a Dataset before deciding to `quarantine` a dataset.) + # + # This could be improved when we drop `pbench-server-prep-shim-002` + # as server `PUT` does not have the same problem. + md5 = Column(String(255), unique=False, nullable=True) + created = Column(DateTime, nullable=False, default=datetime.datetime.now()) + state = Column(Enum(States), unique=False, nullable=False, default=States.UPLOADING) + transition = Column(DateTime, nullable=False, default=datetime.datetime.now()) + + # NOTE: this relationship defines a `dataset` property in `Metadata` + # that refers to the parent `Dataset` object. + metadatas = relationship("Metadata", backref="dataset") + + # Require that the combination of controller and name is unique. + # + # FIXME: I would prefer to check user+controller+name, although + # in practice the chances of controller+name collision are small. + # This is necessary because our current filesystem-based server + # components cannot infer user ownership except by referencing + # this database using filesystem-based tags (controller, name). + # In the future when we change the server components to operate + # entirely by database and messages, we can improve this. + __table_args__ = (UniqueConstraint("controller", "name"), {}) + + @validates("state") + def validate_state(self, key, value): + """Validate that the value provided for the Dataset state is a member + of the States ENUM before it's applied by the SQLAlchemy constructor. + """ + if type(value) is not States: + raise DatasetBadParameterType(value, States) + return value + + @staticmethod + def _render_path(patharg=None, controllerarg=None, namearg=None): + """ + _render_path Process a `path` string and convert it into `controller` + and/or `name` strings. + + This pre-processes the controller and name before they are presented to + a query or constructor. If the calling context has only the full file + path of a dataset, this can extract both "controller" and "name" from + the path. It can also be used to construct either "controller" or + "name", as it will fill in either or both if not already present in + the dict. + + If the path is a symlink (e.g., a "TO-BACKUP" or other Pbench command + link, most likely when a dataset is quarantined due to some consistency + check failure), this code will follow the link in order to construct a + controller name from the original path without needing to make any + possibly fragile assumptions regarding the structure of the symlink + name. + + Args: + patharg: A tarball file path from which the controller (host) + name, the tarball dataset name (basename minus extension), + or both will be derived. + controllerarg: The controller name (hostname) of the dataset; + this is retained if specified, or will be constructed + from "path" if not present. + namearg: The dataset name (file path basename minus ".tar.xz"); + this is retained if specified, or will be constructed from + "path" if not present + + Returns: + A tuple of (controller, name) based on the three arguments + """ + controller_result = controllerarg + name_result = namearg + + if patharg: + path = Path(patharg) + if path.is_symlink(): + path = Path(os.path.realpath(path)) + if not name_result: + name_result = path.name + if name_result.endswith(".tar.xz"): + name_result = name_result[:-7] + if not controller_result: + controller_result = path.parent.name + return controller_result, name_result + + @staticmethod + def create(**kwargs): + """ + create A simple factory method to construct a new Dataset object and + add it to the database. Errors are hidden here, and we return None. + + Args: + kwargs (dict): + "user": The owner of the dataset; defaults to "public". + "path": A tarball file path from which the controller (host) + name, the tarball dataset name (basename minus extension), + or both will be derived. + "controller": The controller name (hostname) of the dataset; + this is retained if specified, or will be constructed + from "path" if not present. + "name": The dataset name (file path basename minus ".tar.xz"); + this is retained if specified, or will be constructed from + "path" if not present. + "state": The initial state of the new dataset. + + Returns: + Dataset: A new Dataset object initialized with the keyword + parameters. (See `__init__` for details.) + """ + try: + dataset = Dataset(**kwargs) + dataset.add() + except Exception: + Dataset.logger.exception( + "Failed create: {}|{}", kwargs.get("controller"), kwargs.get("name") + ) + return None + else: + return dataset + + @staticmethod + def attach(path=None, controller=None, name=None, state=None): + """ + attach Attempt to fetch dataset for the controller and dataset name, + or using a specified file path (see _render_path and the path_init + event listener for details). + + If state is specified, attach will attempt to advance the dataset to + that state. + + Args: + "path": A tarball file path from which the controller (host) + name, the tarball dataset name (basename minus extension), + or both will be derived. + "controller": The controller name (hostname) of the dataset; + this is retained if specified, or will be constructed + from "path" if not present. + "name": The dataset name (file path basename minus ".tar.xz"); + this is retained if specified, or will be constructed from + "path" if not present. + "state": The desired state to advance the dataset. + + Raises: + DatasetSqlError: problem interacting with Database + DatasetNotFound: the specified dataset doesn't exist + DatasetBadParameterType: The state parameter isn't a States ENUM + DatasetTerminalStateViolation: dataset is in terminal state and + can't be advanced + DatasetBadStateTransition: dataset cannot be advanced to the + specified state + + Returns: + Dataset: a dataset object in the desired state (if specified) + """ + # Make sure we have controller and name from path + controller, name = Dataset._render_path(path, controller, name) + try: + dataset = ( + Database.db_session.query(Dataset) + .filter_by(controller=controller, name=name) + .first() + ) + except SQLAlchemyError as e: + Dataset.logger.warning( + "Error attaching {}>{}: {}", controller, name, str(e) + ) + raise DatasetSqlError("attaching", controller, name) from e + + if dataset is None: + Dataset.logger.warning("{}>{} not found", controller, name) + raise DatasetNotFound(controller, name) + elif state: + dataset.advance(state) + return dataset + + def __str__(self): + """ + __str__ Return a string representation of the dataset + + Returns: + string: Representation of the dataset + """ + return f"{self.user}|{self.controller}|{self.name}" + + def advance(self, new_state: States): + """ + advance Modify the state of the Dataset object, if the new_state is + a valid transition from the dataset's current state. + + Args: + new_state (State ENUM): New desired state for the dataset + + Raises: + DatasetBadParameterType: The state parameter isn't a States ENUM + DatasetTerminalStateViolation: The dataset is in a terminal state + that cannot be changed. + DatasetBadStateTransition: The dataset does not support transition + from the current state to the desired state. + """ + if type(new_state) is not States: + raise DatasetBadParameterType(new_state, States) + if self.state not in self.transitions: + self.logger.error( + "Terminal state {} can't advance to {}", self.state, new_state + ) + raise DatasetTerminalStateViolation(self, new_state) + elif new_state not in self.transitions[self.state]: + self.logger.error( + "Current state {} can't advance to {}", self.state, new_state + ) + raise DatasetBadStateTransition(self, new_state) + + # TODO: this would be a good place to generate an audit log + + self.state = new_state + self.transition = datetime.datetime.now() + self.update() + + def add(self): + """ + add Add the Dataset object to the database + """ + try: + Database.db_session.add(self) + Database.db_session.commit() + except Exception as e: + self.logger.error("Can't add {} to DB", str(self)) + Database.db_session.rollback() + raise DatasetSqlError("adding", self.controller, self.name) from e + + def update(self): + """ + update Update the database row with the modified version of the + Dataset object. + """ + try: + Database.db_session.commit() + except Exception as e: + self.logger.error("Can't update {} in DB", str(self)) + Database.db_session.rollback() + raise DatasetSqlError("updating", self.controller, self.name) from e + + +@event.listens_for(Dataset, "init") +def path_init(target, args, kwargs): + """Listen for an init event on a Dataset to process a path before the + SQLAlchemy constructor sees it. + + We want the constructor to see both "controller" and "name" parameters in + the kwargs representing the initial SQL column values. This listener allows + us to provide those values by specifying a file path, which is processed + into controller and name using the internal _render_path() helper. + + We will remove "path" from kwargs so that SQLAlchemy doesn't see it. (This + is an explicitly allowed side effect of the listener architecture.) + """ + if "path" in kwargs: + controller, name = Dataset._render_path( + patharg=kwargs.get("path"), + controllerarg=kwargs.get("controller"), + namearg=kwargs.get("name"), + ) + if "controller" not in kwargs: + kwargs["controller"] = controller + if "name" not in kwargs: + kwargs["name"] = name + del kwargs["path"] + + +class Metadata(Database.Base): + """ Retain secondary information about datasets + + Columns: + id Generated unique ID of table row + dataset_ref Dataset row ID (foreign key) + key Metadata key string + value Metadata value string + """ + + __tablename__ = "metadata" + + # Currently defined metadata keys + REINDEX = "REINDEX" + ARCHIVED = "ARCHIVED" + TARBALL_PATH = "TARBALL_PATH" + + METADATA_KEYS = [REINDEX, ARCHIVED, TARBALL_PATH] + + id = Column(Integer, primary_key=True, autoincrement=True) + key = Column(String(255), unique=False, nullable=False, index=True) + value = Column(String(2048), unique=False, nullable=True) + dataset_ref = Column( + Integer, ForeignKey("dataset.id", ondelete="CASCADE"), nullable=False + ) + + @validates("key") + def validate_key(self, key, value): + """Validate that the value provided for the Metadata key argument is an + allowed name. + """ + if value not in Metadata.METADATA_KEYS: + raise MetadataBadKey(value) + return value + + @staticmethod + def create(**kwargs): + if "dataset" not in kwargs: + raise MetadataMissingParameter("dataset") + dataset = kwargs.get("dataset") + if type(dataset) is not Dataset: + raise DatasetBadParameterType(dataset, Dataset) + + try: + meta = Metadata(**kwargs) + meta.add(dataset) + except Exception: + Metadata.logger.exception( + "Failed create: {}>>{}", kwargs.get("dataset"), kwargs.get("key") + ) + return None + else: + return meta + + @staticmethod + def get(dataset: Dataset, key: str): + try: + meta = ( + Database.db_session.query(Metadata) + .filter_by(dataset=dataset, key=key) + .first() + ) + except SQLAlchemyError as e: + Metadata.logger.exception("Can't get {}>>{} from DB", dataset, key) + raise MetadataSqlError("getting", dataset, key) from e + else: + if meta is None: + raise MetadataNotFound(dataset, key) + return meta + + @staticmethod + def remove(dataset: Dataset, key: str): + """ + remove Remove a metadata key from the dataset + + Args: + dataset (Dataset): Dataset with which key is associated + key (string): Name of metadata key to remove + + Raises: + DatasetSqlError: Something went wrong + """ + try: + Database.db_session.query(Metadata).filter_by( + dataset=dataset, key=key + ).delete() + Database.db_session.commit() + except SQLAlchemyError as e: + Metadata.logger.exception("Can't remove {}>>{} from DB", dataset, key) + raise MetadataSqlError("deleting", dataset, key) from e + + def __str__(self): + return f"{self.dataset}>>{self.key}" + + def add(self, dataset: Dataset): + """ + add Add the Metadata object to the dataset + """ + if type(dataset) is not Dataset: + raise DatasetBadParameterType(dataset, Dataset) + + try: + Metadata.get(dataset, self.key) + except MetadataNotFound: + pass + else: + raise MetadataDuplicateKey(dataset, self.key) + + try: + dataset.metadatas.append(self) + Database.db_session.add(self) + Database.db_session.commit() + except Exception as e: + Metadata.logger.exception("Can't add {}>>{} to DB", dataset, self.key) + Database.db_session.rollback() + dataset.metadatas.remove(self) + raise MetadataSqlError("adding", dataset, self.key) from e + + def update(self): + """ + update Update the database with the modified Metadata. + """ + try: + Database.db_session.commit() + except Exception as e: + Metadata.logger.exception("Can't update {} in DB", self) + Database.db_session.rollback() + raise MetadataSqlError("updating", self.dataset, self.key) from e + + def delete(self): + """ + delete Remove the Metadata from the database. + """ + try: + Database.db_session.delete(self) + Database.db_session.commit() + except Exception as e: + Metadata.logger.exception("Can't delete {} from DB", self) + Database.db_session.rollback() + raise MetadataSqlError("deleting", self.dataset, self.key) from e + + +@event.listens_for(Metadata, "init") +def check_required(target, args, kwargs): + """Listen for an init event on Metadata to verify parameters before the + SQLAlchemy constructor gets control. + + We want the constructor to see both "key" and "value" parameters; if either + isn't present, give a simpler and more useful error message rather than the + internal SQL constraint failures that would result. + """ + if "key" not in kwargs: + raise MetadataMissingParameter("key") + if "value" not in kwargs: + raise MetadataMissingKeyValue(kwargs.get("key")) diff --git a/lib/pbench/server/indexing_tarballs.py b/lib/pbench/server/indexing_tarballs.py index 3439aee3eb..e1399b4c83 100644 --- a/lib/pbench/server/indexing_tarballs.py +++ b/lib/pbench/server/indexing_tarballs.py @@ -19,6 +19,13 @@ es_index, VERSION, ) +from pbench.server.database.models.tracker import ( + Dataset, + States, + Metadata, + DatasetNotFound, + DatasetTransitionError, +) from pbench.server.report import Report from pbench.server.utils import rename_tb_link, quarantine @@ -310,18 +317,35 @@ def sighup_handler(*args): ) idxctx.logger.info("Starting {} (size {:d})", tb, size) - + dataset = None ptb = None try: + path = os.path.realpath(tb) + # "Open" the tar ball represented by the tar ball object idxctx.logger.debug("open tar ball") ptb = PbenchTarBall( - idxctx, - os.path.realpath(tb), - tmpdir, - Path(self.incoming, controller), + idxctx, path, tmpdir, Path(self.incoming, controller), ) + try: + dataset = Dataset.attach( + path=path, state=States.INDEXING, + ) + except DatasetNotFound: + idxctx.logger.warn( + "Unable to locate Dataset {}", path, + ) + except DatasetTransitionError as e: + # TODO: This means the Dataset is known, but not in a + # state where we'd expect to be indexing it. So what do + # we do with it? (Note: this is where an audit log will + # be handy; i.e., how did we get here?) For now, just + # let it go. + idxctx.logger.warn( + "Unable to advance dataset state: {}", str(e) + ) + # Construct the generator for emitting all actions. The # `idxctx` dictionary is passed along to each generator so # that it can add its context for error handling to the @@ -393,6 +417,18 @@ def sighup_handler(*args): retries, ) tb_res = error_code["OP_ERROR" if failures > 0 else "OK"] + finally: + if dataset: + try: + dataset.advance( + States.INDEXED + if tb_res.success + else States.QUARANTINED + ) + Metadata.remove(dataset, Metadata.REINDEX) + except DatasetTransitionError: + idxctx.logger.exception("Dataset state error") + try: ie_len = ie_filepath.stat().st_size except FileNotFoundError: diff --git a/lib/pbench/server/utils.py b/lib/pbench/server/utils.py index 17b6998d4b..44273ca611 100644 --- a/lib/pbench/server/utils.py +++ b/lib/pbench/server/utils.py @@ -2,6 +2,8 @@ import sys import shutil +from pbench.server.database.models.tracker import Dataset, States, DatasetNotFound + def rename_tb_link(tb, dest, logger): try: @@ -73,6 +75,13 @@ def quarantine(dest, logger, *files): if not os.path.exists(afile) and not os.path.islink(afile): continue try: + # If the file we're moving is a tarball, update the dataset + # state. (If it's the associated MD5 file, skip that.) + if str(afile).endswith(".tar.xz"): + try: + Dataset.attach(path=afile, state=States.QUARANTINED) + except DatasetNotFound: + logger.exception("quarantine dataset {} not found", afile) shutil.move(afile, os.path.join(dest, os.path.basename(afile))) except Exception: logger.exception( diff --git a/lib/pbench/test/unit/server/conftest.py b/lib/pbench/test/unit/server/conftest.py index 974fa94e51..08a83ccd1a 100644 --- a/lib/pbench/test/unit/server/conftest.py +++ b/lib/pbench/test/unit/server/conftest.py @@ -93,9 +93,9 @@ def server_config(pytestconfig, monkeypatch): @pytest.fixture def client(server_config): + """A test client for the app.""" app = create_app(server_config) - """A test client for the app.""" app_client = app.test_client() app_client.logger = app.logger app_client.config = app.config diff --git a/lib/pbench/test/unit/server/test_requests.py b/lib/pbench/test/unit/server/test_requests.py index a040cf0bba..f47b875c71 100644 --- a/lib/pbench/test/unit/server/test_requests.py +++ b/lib/pbench/test/unit/server/test_requests.py @@ -4,6 +4,7 @@ from pathlib import Path from werkzeug.utils import secure_filename +from pbench.server.database.models.tracker import Dataset, States class TestHostInfo: @@ -139,13 +140,13 @@ def test_empty_upload(client, pytestconfig, caplog, server_config): def test_upload(client, pytestconfig, caplog, server_config): filename = "log.tar.xz" datafile = Path("./lib/pbench/test/unit/server/fixtures/upload/", filename) - + controller = socket.gethostname() with open(f"{datafile}.md5") as md5sum_check: md5sum = md5sum_check.read() with open(datafile, "rb") as data_fp: response = client.put( - f"{server_config.rest_uri}/upload/ctrl/{socket.gethostname()}", + f"{server_config.rest_uri}/upload/ctrl/{controller}", data=data_fp, headers={"filename": filename, "Content-MD5": md5sum}, ) @@ -160,5 +161,13 @@ def test_upload(client, pytestconfig, caplog, server_config): f"receive_dir = '{receive_dir}', filename = '{filename}'," f" sfilename = '{sfilename}'" ) + + dataset = Dataset.attach(controller=controller, path=filename) + assert dataset is not None + assert dataset.md5 == md5sum + assert dataset.controller == controller + assert dataset.name == "log" + assert dataset.state == States.UPLOADED + for record in caplog.records: assert record.levelname not in ("WARNING", "ERROR", "CRITICAL") diff --git a/lib/pbench/test/unit/server/test_state_tracker.py b/lib/pbench/test/unit/server/test_state_tracker.py new file mode 100644 index 0000000000..e8c1eda279 --- /dev/null +++ b/lib/pbench/test/unit/server/test_state_tracker.py @@ -0,0 +1,258 @@ +import pytest +from pbench.server.database.models.tracker import ( + Dataset, + States, + Metadata, + DatasetBadParameterType, + DatasetBadStateTransition, + DatasetTerminalStateViolation, + DatasetNotFound, + MetadataNotFound, + MetadataBadKey, + MetadataMissingKeyValue, + MetadataDuplicateKey, +) + + +class TestStateTracker: + def test_state_enum(self): + """ Test the States ENUM properties + """ + assert len(States.__members__) == 9 + for n, s in States.__members__.items(): + assert str(s) == s.friendly + assert s.mutating == ( + "ing" in s.friendly + ), f"Enum {n} name and state don't match" + + def test_construct(self): + """ Test dataset contructor + """ + ds = Dataset(user="drb", controller="frodo", name="fio") + ds.add() + assert ds.user == "drb" + assert ds.controller == "frodo" + assert ds.name == "fio" + assert ds.state == States.UPLOADING + assert ds.md5 is None + assert ds.created <= ds.transition + assert ds.id is not None + assert "drb|frodo|fio" == str(ds) + + def test_construct_bad_state(self): + """Test with a non-States state value + """ + with pytest.raises(DatasetBadParameterType): + Dataset(controller="potter", name="harry", state="notStates") + + def test_attach_exists(self): + """ Test that we can attach to a dataset + """ + ds1 = Dataset( + user="drb", controller="frodo", name="fido", state=States.INDEXING + ) + ds1.add() + + ds2 = Dataset.attach(controller="frodo", name="fido", state=States.INDEXED) + assert ds2.user == ds1.user + assert ds2.controller == ds1.controller + assert ds2.name == ds1.name + assert ds2.state == States.INDEXED + assert ds2.md5 is ds1.md5 + assert ds2.id is ds1.id + + def test_attach_none(self): + """ Test expected failure when we try to attach to a dataset that + does not exist. + """ + with pytest.raises(DatasetNotFound): + Dataset.attach(controller="frodo", name="venus", state=States.UPLOADING) + + def test_attach_controller_path(self): + """ Test that we can attach using controller and name to a + dataset created by file path. + """ + ds1 = Dataset( + user="pete", path="/foo/frodo/rover.tar.xz", state=States.INDEXING, + ) + ds1.add() + + ds2 = Dataset.attach(controller="frodo", name="rover") + assert ds2.user == ds1.user + assert ds2.controller == ds1.controller + assert ds2.name == ds1.name + assert ds2.state == States.INDEXING + assert ds2.md5 is ds1.md5 + assert ds2.id is ds1.id + + def test_attach_filename(self): + """ Test that we can create a dataset using the full tarball + file path. + """ + ds1 = Dataset( + user="webb", path="/foo/bilbo/rover.tar.xz", state=States.QUARANTINED + ) + ds1.add() + + ds2 = Dataset.attach(controller="bilbo", name="rover") + assert ds2.user == ds1.user + assert ds2.controller == ds1.controller + assert ds2.name == ds1.name + assert ds2.state == States.QUARANTINED + assert ds2.md5 is ds1.md5 + assert ds2.id is ds1.id + + def test_advanced_good(self): + """ Test advancing the state of a dataset + """ + ds = Dataset(user="drb", controller="frodo", name="precious") + ds.add() + ds.advance(States.UPLOADED) + assert ds.state == States.UPLOADED + assert ds.created <= ds.transition + + def test_advanced_bad_state(self): + """Test with a non-States state value + """ + ds = Dataset(controller="me", name="too") + ds.add() + with pytest.raises(DatasetBadParameterType): + ds.advance("notStates") + + def test_advanced_illegal(self): + """ Test that we can't advance to a state that's not a + successor to the initial state. + """ + ds = Dataset(user="drb", controller="sam", name="fio") + ds.add() + with pytest.raises(DatasetBadStateTransition): + ds.advance(States.EXPIRED) + + def test_advanced_terminal(self): + """ Test that we can't advance from a terminal state + """ + ds = Dataset(user="drb", controller="gimli", name="fio", state=States.EXPIRED) + ds.add() + with pytest.raises(DatasetTerminalStateViolation): + ds.advance(States.UPLOADING) + + def test_lifecycle(self): + """ Advance a dataset through the entire lifecycle using the state + transition dict. + """ + ds = Dataset(user="dave", controller="bilbo", name="Fred") + ds.add() + assert ds.state == States.UPLOADING + beenthere = [ds.state] + while ds.state in Dataset.transitions: + advances = Dataset.transitions[ds.state] + for n in advances: + if n not in beenthere: + next = n + break + else: + break # avoid infinite reindex loop! + beenthere.append(next) + ds.advance(next) + assert ds.state == next + lifecycle = ",".join([s.name for s in beenthere]) + assert ( + lifecycle + == "UPLOADING,UPLOADED,UNPACKING,UNPACKED,INDEXING,INDEXED,EXPIRING,EXPIRED" + ) + + def test_metadata(self): + """ Various tests on Metadata keys + """ + # See if we can create a metadata row + ds = Dataset.create(user="user", controller="controller", name="name") + assert ds.metadatas == [] + m = Metadata.create(key=Metadata.REINDEX, value="TRUE", dataset=ds) + assert m is not None + assert ds.metadatas == [m] + + # Try to get it back + m1 = Metadata.get(ds, Metadata.REINDEX) + assert m1.key == m.key + assert m1.value == m.value + assert m.id == m1.id + assert m.dataset_ref == m1.dataset_ref + + # Check the str() + assert "user|controller|name>>REINDEX" == str(m) + + # Try to get a metadata key that doesn't exist + with pytest.raises(MetadataNotFound) as exc: + Metadata.get(ds, Metadata.TARBALL_PATH) + assert exc.value.dataset == ds + assert exc.value.key == Metadata.TARBALL_PATH + + # Try to remove a metadata key that doesn't exist (No-op) + Metadata.remove(ds, Metadata.TARBALL_PATH) + + # Try to create a metadata with a bad key + badkey = "THISISNOTTHEKEYYOURELOOKINGFOR" + with pytest.raises(MetadataBadKey) as exc: + Metadata(key=badkey, value=None) + assert exc.value.key == badkey + + # Try to create a key without a value + with pytest.raises(MetadataMissingKeyValue): + Metadata(key=Metadata.REINDEX) + + # Try to add a duplicate metadata key + with pytest.raises(MetadataDuplicateKey) as exc: + m1 = Metadata(key=Metadata.REINDEX, value="IRRELEVANT") + m1.add(ds) + assert exc.value.key == Metadata.REINDEX + assert exc.value.dataset == ds + assert ds.metadatas == [m] + + # Try to add a Metadata key to something that's not a dataset + with pytest.raises(DatasetBadParameterType) as exc: + m1 = Metadata(key=Metadata.TARBALL_PATH, value="DONTCARE") + m1.add("foobar") + assert exc.value.bad_value == "foobar" + assert exc.value.expected_type == Dataset + + # Try to create a Metadata with a bad value for the dataset + with pytest.raises(DatasetBadParameterType) as exc: + m1 = Metadata.create(key=Metadata.REINDEX, value="TRUE", dataset=[ds]) + assert exc.value.bad_value == [ds] + assert exc.value.expected_type == Dataset + + # Try to update the metadata key + m.value = "False" + m.update() + m1 = Metadata.get(ds, Metadata.REINDEX) + assert m.id == m1.id + assert m.dataset_ref == m1.dataset_ref + assert m.key == m1.key + assert m.value == "False" + + # Delete the key and make sure its gone + m.delete() + with pytest.raises(MetadataNotFound) as exc: + Metadata.get(ds, Metadata.REINDEX) + assert exc.value.dataset == ds + assert exc.value.key == Metadata.REINDEX + assert ds.metadatas == [] + + def test_metadata_remove(self): + """ Test that we can remove a Metadata key + """ + ds = Dataset.create(user="user1", controller="controller", name="other") + assert ds.metadatas == [] + m = Metadata(key=Metadata.ARCHIVED, value="TRUE") + m.add(ds) + assert ds.metadatas == [m] + + Metadata.remove(ds, Metadata.ARCHIVED) + assert ds.metadatas == [] + with pytest.raises(MetadataNotFound) as exc: + Metadata.get(ds, Metadata.ARCHIVED) + assert exc.value.dataset == ds + assert exc.value.key == Metadata.ARCHIVED + + Metadata.remove(ds, Metadata.REINDEX) + assert ds.metadatas == [] diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py index 52fb08cd5b..7f6923c0b0 100644 --- a/lib/pbench/test/unit/server/test_user_auth.py +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -60,7 +60,7 @@ def test_registration_missing_fields(client, server_config): }, ) data = response.json - assert data["message"] == "Missing firstName field" + assert data["message"] == "Missing first_name field" assert response.content_type == "application/json" assert response.status_code == 400 diff --git a/server/bin/gold/test-11.txt b/server/bin/gold/test-11.txt index 6c1c4806bd..a6e28be071 100644 --- a/server/bin/gold/test-11.txt +++ b/server/bin/gold/test-11.txt @@ -159,6 +159,8 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-sync-satellite drwxrwxr-x - logs/pbench-sync-satellite/ONE drwxrwxr-x - logs/pbench-sync-satellite/ONE/run-1970-01-01T00:00:42-UTC @@ -231,6 +233,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-sync-satellite/ONE/run-1970-01-01T00:00:42-UTC/controller/fail-checks.log pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: FAILED ----- pbench-sync-satellite/ONE/run-1970-01-01T00:00:42-UTC/controller/fail-checks.log @@ -266,3 +270,6 @@ run-1970-01-01T00:00:42-UTC: Total 2 files processed, with 1 md5 failures and 0 ssh pbench-satellite.example.com /var/tmp/pbench-test-server/test-11/opt/pbench-server-satellite/bin/pbench-sync-package-tarballs ssh pbench-satellite.example.com /var/tmp/pbench-test-server/test-11/opt/pbench-server-satellite/bin/pbench-satellite-state-change /var/tmp/pbench-test-server/test-11/pbench-satellite/archive/fs-version-001 --- test-execution.log file contents ++++ SqliteDB Datasets +public|ONE::controller|fio__2016-08-16_22:03:11|UPLOADED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-16.txt b/server/bin/gold/test-16.txt index 2c1812055c..979e37e95b 100644 --- a/server/bin/gold/test-16.txt +++ b/server/bin/gold/test-16.txt @@ -227,6 +227,8 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs-small -rw-rw-r-- 202 logs/pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error -rw-rw-r-- 1970 logs/pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.log @@ -283,6 +285,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error run-1970-01-01T00:00:42-UTC: symlink target for /var/tmp/pbench-test-server/test-16/pbench/archive/fs-version-001/controller00/TO-UNPACK/benchmark-doesnotexist_1970-01-01T00:00:00.tar.xz does not exist ----- pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error @@ -303,3 +307,8 @@ run-1970-01-01T00:00:42-UTC: Processed 3 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|controller00|benchmark-result-large_1970-01-01T00:00:00|UNPACKED| +public|controller01|benchmark-result-medium_1970-01-01T00:00:00|UNPACKED| +public|controller02|benchmark-result-small_1970-01-01T00:00:00|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-17.txt b/server/bin/gold/test-17.txt index a21a870171..c5374453f8 100644 --- a/server/bin/gold/test-17.txt +++ b/server/bin/gold/test-17.txt @@ -227,6 +227,8 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs-small -rw-rw-r-- 202 logs/pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error -rw-rw-r-- 1969 logs/pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.log @@ -283,6 +285,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error run-1970-01-01T00:00:42-UTC: symlink target for /var/tmp/pbench-test-server/test-17/pbench/archive/fs-version-001/controller00/TO-UNPACK/benchmark-doesnotexist_1970-01-01T00:00:00.tar.xz does not exist ----- pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error @@ -303,3 +307,8 @@ run-1970-01-01T00:00:42-UTC: Processed 3 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|controller00|benchmark-result-large_1970-01-01T00:00:00|UNPACKED| +public|controller01|benchmark-result-medium_1970-01-01T00:00:00|UNPACKED| +public|controller02|benchmark-result-small_1970-01-01T00:00:00|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-29.1.txt b/server/bin/gold/test-29.1.txt index 44f0a8191b..f69cdc370e 100644 --- a/server/bin/gold/test-29.1.txt +++ b/server/bin/gold/test-29.1.txt @@ -89,6 +89,8 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-reindex +-rw-rw-r-- 0 logs/pbench-reindex/pbench-reindex.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -141,6 +143,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-reindex/pbench-reindex.log +----- pbench-reindex/pbench-reindex.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs diff --git a/server/bin/gold/test-29.2.txt b/server/bin/gold/test-29.2.txt index 2ee9e4c7df..4e90a54dab 100644 --- a/server/bin/gold/test-29.2.txt +++ b/server/bin/gold/test-29.2.txt @@ -142,6 +142,10 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 943 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-reindex +-rw-rw-r-- 0 logs/pbench-reindex/pbench-reindex.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -208,7 +212,14 @@ end-1970-01-01T00:00:42-UTC: archive hierarchy: /var/tmp/pbench-test-server/test 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-reindex/pbench-reindex.log +----- pbench-reindex/pbench-reindex.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlr-in-range|tarball_re-index_1970.02.14T12.42.42|INDEXED| REINDEX = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-29.3.txt b/server/bin/gold/test-29.3.txt index 43a1220758..dbd55e3510 100644 --- a/server/bin/gold/test-29.3.txt +++ b/server/bin/gold/test-29.3.txt @@ -142,6 +142,8 @@ drwxrwxr-x - logs drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 0 logs/pbench-audit-server/pbench-audit-server.error -rw-rw-r-- 943 logs/pbench-audit-server/pbench-audit-server.log +drwxrwxr-x - logs/pbench-reindex +-rw-rw-r-- 0 logs/pbench-reindex/pbench-reindex.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -208,6 +210,8 @@ end-1970-01-01T00:00:42-UTC: archive hierarchy: /var/tmp/pbench-test-server/test 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-audit-server.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-audit-server/pbench-audit-server.log ++++++ pbench-reindex/pbench-reindex.log +----- pbench-reindex/pbench-reindex.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs diff --git a/server/bin/gold/test-5.2.txt b/server/bin/gold/test-5.2.txt index 90aa58fb66..83f511e711 100644 --- a/server/bin/gold/test-5.2.txt +++ b/server/bin/gold/test-5.2.txt @@ -686,6 +686,8 @@ drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 7805 logs/pbench-index/pbench-index.log drwxrwxr-x - logs/pbench-server-prep-shim-002 -rw-rw-r-- 2988 logs/pbench-server-prep-shim-002/pbench-server-prep-shim-002.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-sync-satellite drwxrwxr-x - logs/pbench-sync-satellite/ONE -rw-rw-r-- 219 logs/pbench-sync-satellite/pbench-sync-satellite.error @@ -1015,6 +1017,8 @@ run-1970-01-01T00:00:42-UTC: Processed 10 result tar balls, 7 successfully (0 pa 1970-01-01T00:00:42.000000 DEBUG pbench-server-prep-shim-002.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, retries: 0) 1970-01-01T00:00:42.000000 DEBUG pbench-server-prep-shim-002.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-server-prep-shim-002/pbench-server-prep-shim-002.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-sync-satellite/pbench-sync-satellite.error pbench-sync-satellite: completing satellite state changes ... (/var/tmp/pbench-test-server/test-5.2/pbench-local/logs/pbench-sync-satellite/ONE/change_state.log) pbench-sync-satellite: completed satellite state changes @@ -1104,3 +1108,18 @@ selinux.restorecon('/var/tmp/pbench-test-server/test-5.2/pbench/archive/fs-versi ssh pbench-satellite.example.com /var/tmp/pbench-test-server/test-5.2/opt/pbench-server-satellite/bin/pbench-sync-package-tarballs ssh pbench-satellite.example.com /var/tmp/pbench-test-server/test-5.2/opt/pbench-server-satellite/bin/pbench-satellite-state-change /var/tmp/pbench-test-server/test-5.2/pbench-satellite/archive/fs-version-001 --- test-execution.log file contents ++++ SqliteDB Datasets +public|ONE::controllerA|tarball-simple1_1970-01-01T00:42:00|UNPACKED| ARCHIVED = True +public|ONE::controllerB|tarball-simple2_1970-01-01T00:41:00|UNPACKED| ARCHIVED = True +public|ONE::controllerC|tarball-simple0-prefix_1970-01-01T00:42:00|UNPACKED| ARCHIVED = True +public|controller-b-with-prefixes|tarball-0_1970.01.01T00.42.00|UNPACKED| ARCHIVED = True +public|controller-b-with-prefixes|tarball-w-dot-prefix_1970.01.01T00.42.00|UNPACKED| ARCHIVED = True +public|controller-b-with-prefixes|tarball-w-prefix-dot_1970.01.01T00.42.00|UNPACKED| ARCHIVED = True +public|controller-d-duplicate|DUPLICATE__NAME.tarball-duplicate_1970.01.01T00.42.00|QUARANTINED| +public|controller-d-duplicate|tarball-duplicate_1970.01.01T00.42.00|QUARANTINED| ARCHIVED = True +public|controller-d-duplicate|tarball-that-does-not-exist_1970.01.01T00.42.00|QUARANTINED| +public|controller-e-bad-md5|tarball-bad-md5_1970.01.01T00.42.00|QUARANTINED| +public|controller-f-bad-md5|tarball-bad-md5-1_1970.01.01T00.42.00|QUARANTINED| +public|controller-g-normal|tarball-normal_1970.01.01T00.42.00|UNPACKED| ARCHIVED = True +public|controller-h-bad-md5|tarball-bad-md5_1970.01.01T00.42.00|QUARANTINED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.10.txt b/server/bin/gold/test-6.10.txt index b00c2513fc..d463890c59 100644 --- a/server/bin/gold/test-6.10.txt +++ b/server/bin/gold/test-6.10.txt @@ -127,6 +127,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2122 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -197,7 +199,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UPLOADED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.11.txt b/server/bin/gold/test-6.11.txt index 8fe1849f09..4062622610 100644 --- a/server/bin/gold/test-6.11.txt +++ b/server/bin/gold/test-6.11.txt @@ -127,6 +127,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2514 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -199,7 +201,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|INDEXING| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.12.txt b/server/bin/gold/test-6.12.txt index 2c0dd277a2..16bd142ebc 100644 --- a/server/bin/gold/test-6.12.txt +++ b/server/bin/gold/test-6.12.txt @@ -127,6 +127,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2523 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -200,7 +202,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|INDEXED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.13.txt b/server/bin/gold/test-6.13.txt index 03fd5952f5..24913393e7 100644 --- a/server/bin/gold/test-6.13.txt +++ b/server/bin/gold/test-6.13.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 3220 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -204,7 +206,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.14.txt b/server/bin/gold/test-6.14.txt index eda5297a27..9754aefc0d 100644 --- a/server/bin/gold/test-6.14.txt +++ b/server/bin/gold/test-6.14.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 3220 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -204,7 +206,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.15.txt b/server/bin/gold/test-6.15.txt index 8b8ad8dfef..d3e3ee9276 100644 --- a/server/bin/gold/test-6.15.txt +++ b/server/bin/gold/test-6.15.txt @@ -127,6 +127,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2122 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -198,7 +200,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.16.txt b/server/bin/gold/test-6.16.txt index 7a86333a07..c5eda10614 100644 --- a/server/bin/gold/test-6.16.txt +++ b/server/bin/gold/test-6.16.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 3975 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -211,7 +213,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|client_error_other|UNPACKED| ARCHIVED = True +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.17.txt b/server/bin/gold/test-6.17.txt index 1949c61544..ab10f4f120 100644 --- a/server/bin/gold/test-6.17.txt +++ b/server/bin/gold/test-6.17.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 3926 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -211,7 +213,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +public|foo|get_object_exception|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.4.txt b/server/bin/gold/test-6.4.txt index 1355702224..55734c6aea 100644 --- a/server/bin/gold/test-6.4.txt +++ b/server/bin/gold/test-6.4.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 3442 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -210,7 +212,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UPLOADED| ARCHIVED = True +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|UPLOADED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.5.txt b/server/bin/gold/test-6.5.txt index 0afde0b0bf..9834c9e6b1 100644 --- a/server/bin/gold/test-6.5.txt +++ b/server/bin/gold/test-6.5.txt @@ -132,6 +132,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 4720 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -216,7 +218,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.6.txt b/server/bin/gold/test-6.6.txt index c52dbad6dd..9bc105d215 100644 --- a/server/bin/gold/test-6.6.txt +++ b/server/bin/gold/test-6.6.txt @@ -129,6 +129,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2877 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -202,7 +204,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|QUARANTINED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.7.txt b/server/bin/gold/test-6.7.txt index 2fbf020e3d..eecbbb8731 100644 --- a/server/bin/gold/test-6.7.txt +++ b/server/bin/gold/test-6.7.txt @@ -122,6 +122,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2124 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -189,7 +191,13 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|QUARANTINED| +public|foo|pbench-user-benchmark__2016-08-24_21:32:01|QUARANTINED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-6.9.txt b/server/bin/gold/test-6.9.txt index 8cafc92f0d..b936be0baf 100644 --- a/server/bin/gold/test-6.9.txt +++ b/server/bin/gold/test-6.9.txt @@ -126,6 +126,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-backup-tarballs -rw-rw-r-- 2129 logs/pbench-backup-tarballs/pbench-backup-tarballs.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -196,7 +198,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 DEBUG pbench-backup-tarballs.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) 1970-01-01T00:00:42.000000 INFO pbench-backup-tarballs.pbench-backup-tarballs main -- end-run-1970-01-01T00:00:42-UTC ----- pbench-backup-tarballs/pbench-backup-tarballs.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|foo|fio__2016-08-18_15:47:09|UNPACKED| ARCHIVED = True +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.1.txt b/server/bin/gold/test-7.1.txt index 4fd44953f7..6d09b2c7fd 100644 --- a/server/bin/gold/test-7.1.txt +++ b/server/bin/gold/test-7.1.txt @@ -234,6 +234,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 265 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 2294 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -309,6 +311,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 0 (skipped 1) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -323,3 +327,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|controller|test_7.1_1970.01.01T00.00.00|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.10.txt b/server/bin/gold/test-7.10.txt index 434f5be03d..be5067c033 100644 --- a/server/bin/gold/test-7.10.txt +++ b/server/bin/gold/test-7.10.txt @@ -10735,6 +10735,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 10315 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3643 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 883 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -10874,6 +10876,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -10888,3 +10892,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|dhcp31-44|uperf_uperftest_2018.02.02T20.58.00|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.11.txt b/server/bin/gold/test-7.11.txt index d51c9b9a00..bd1b6d6dea 100644 --- a/server/bin/gold/test-7.11.txt +++ b/server/bin/gold/test-7.11.txt @@ -6013,6 +6013,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 5958 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3616 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 856 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -6135,6 +6137,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -6149,3 +6153,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|dhcp31-44|fio_rw_2018.02.01T22.40.57|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.12.txt b/server/bin/gold/test-7.12.txt index c60c966b82..889f103278 100644 --- a/server/bin/gold/test-7.12.txt +++ b/server/bin/gold/test-7.12.txt @@ -4992,6 +4992,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 9372 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3701 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 940 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -5125,6 +5127,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -5139,3 +5143,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|EC2::ip-172-31-52-154|pbench-user-benchmark__2018.02.05T20.35.36|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.13.txt b/server/bin/gold/test-7.13.txt index 00b961daae..56b36b089c 100644 --- a/server/bin/gold/test-7.13.txt +++ b/server/bin/gold/test-7.13.txt @@ -11570,6 +11570,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 4620 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3719 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 958 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -11685,6 +11687,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -11699,3 +11703,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|b03-h01-1029p|pbench-user-benchmark_mbruzek-test-2_2018.04.10T19.01.19|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.14.txt b/server/bin/gold/test-7.14.txt index 9c42f624ca..9d87ae6204 100644 --- a/server/bin/gold/test-7.14.txt +++ b/server/bin/gold/test-7.14.txt @@ -8721,6 +8721,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 39130 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3719 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 958 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -8964,6 +8966,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -8978,3 +8982,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|b03-h01-1029p|pbench-user-benchmark_mbruzek-test-2_2018.04.10T19.01.19|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.15.txt b/server/bin/gold/test-7.15.txt index 2ec4df51dc..15b2b502a3 100644 --- a/server/bin/gold/test-7.15.txt +++ b/server/bin/gold/test-7.15.txt @@ -9370,6 +9370,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 30144 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3624 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 862 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9589,6 +9591,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9603,3 +9607,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|master_40gb|uperf__2016-10-06_16:34:03|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.16.txt b/server/bin/gold/test-7.16.txt index 81e9eed74d..49dff6f81d 100644 --- a/server/bin/gold/test-7.16.txt +++ b/server/bin/gold/test-7.16.txt @@ -11171,6 +11171,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 19507 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3697 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 937 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -11342,6 +11344,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -11356,3 +11360,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|rhel8-4|uperf_rhel8_4.18.0-18.el8_40gb_pass_2018.10.04T06.53.43|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.17.txt b/server/bin/gold/test-7.17.txt index cae8a8e5d7..ad480cc1b6 100644 --- a/server/bin/gold/test-7.17.txt +++ b/server/bin/gold/test-7.17.txt @@ -3733,6 +3733,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 9006 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3712 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 982 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -3865,6 +3867,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -3879,3 +3883,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ansible-host|pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.18.txt b/server/bin/gold/test-7.18.txt index bf1904be54..6a819e958e 100644 --- a/server/bin/gold/test-7.18.txt +++ b/server/bin/gold/test-7.18.txt @@ -240,6 +240,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 265 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 2629 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1271 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -317,6 +319,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 0 (skipped 1) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -333,3 +337,7 @@ run-1970-01-01T00:00:42-UTC: Processed 2 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|bad-controller|pbench-user-benchmark__2018.02.05T20.35.36|QUARANTINED| +public|bad-controller|test_7.18_2018.02.05T15.31.08|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.19.txt b/server/bin/gold/test-7.19.txt index 579b44f125..094dda865f 100644 --- a/server/bin/gold/test-7.19.txt +++ b/server/bin/gold/test-7.19.txt @@ -6974,6 +6974,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3611 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 4242 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1093 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -7086,6 +7088,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -7100,3 +7104,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|perf122|trafficgen_basic-forwarding-example_tg:trex-profile_pf:forwarding_test.json_ml:5_tt:bs__2019-08-27T14:58:38|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.2.0.txt b/server/bin/gold/test-7.2.0.txt index ac93a20a1f..3995bd55c2 100644 --- a/server/bin/gold/test-7.2.0.txt +++ b/server/bin/gold/test-7.2.0.txt @@ -233,6 +233,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 265 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 2320 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 870 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -308,6 +310,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 0 (skipped 1) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -322,3 +326,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|controller|test_7.2.0_1970.01.01T00.42.00|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.2.1.txt b/server/bin/gold/test-7.2.1.txt index d9a0597607..b8e4805843 100644 --- a/server/bin/gold/test-7.2.1.txt +++ b/server/bin/gold/test-7.2.1.txt @@ -977,6 +977,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 265 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 2375 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 861 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1052,6 +1054,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 0 (skipped 1) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1066,3 +1070,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|controller|uperf__2016-10-06_16:34:03|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.20.txt b/server/bin/gold/test-7.20.txt index 538c058d96..1e978b37c8 100644 --- a/server/bin/gold/test-7.20.txt +++ b/server/bin/gold/test-7.20.txt @@ -2518,6 +2518,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3366 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3612 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1102 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -2629,6 +2631,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -2644,3 +2648,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|fio_mock_2020.02.27T22.16.14|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.21.txt b/server/bin/gold/test-7.21.txt index ec5c3c275e..f503fa3e02 100644 --- a/server/bin/gold/test-7.21.txt +++ b/server/bin/gold/test-7.21.txt @@ -9804,6 +9804,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 5652 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 6749 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1788 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9952,6 +9954,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 2 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9970,3 +9974,7 @@ run-1970-01-01T00:00:42-UTC: Processed 2 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|trafficgen_mock_2020.02.28T19.49.39|INDEXED| +public|ctlrA|trafficgen_mock_2020.02.28T20.04.29|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.22.txt b/server/bin/gold/test-7.22.txt index 6ac2fd4c01..d0298b7f36 100644 --- a/server/bin/gold/test-7.22.txt +++ b/server/bin/gold/test-7.22.txt @@ -1281,6 +1281,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3376 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3923 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1118 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1393,6 +1395,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1408,3 +1412,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|linpack_mock_2020.02.28T19.10.55|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.23.txt b/server/bin/gold/test-7.23.txt index f6d1990b6e..24be72212d 100644 --- a/server/bin/gold/test-7.23.txt +++ b/server/bin/gold/test-7.23.txt @@ -3018,6 +3018,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3366 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3612 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1075 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -3129,6 +3131,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -3144,3 +3148,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|fio_mock_2020.01.19T00.18.06|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.24.txt b/server/bin/gold/test-7.24.txt index 52ed57b103..673822139d 100644 --- a/server/bin/gold/test-7.24.txt +++ b/server/bin/gold/test-7.24.txt @@ -1346,6 +1346,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 5415 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 6011 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 1444 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1489,6 +1491,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 2 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -1505,3 +1509,7 @@ run-1970-01-01T00:00:42-UTC: Processed 2 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|pbench-user-benchmark_1000_80_latency_perf_2021.01.28T17.22.57|INDEXED| +public|ctlrA|pbench-user-benchmark_Maridb_tuned_TP_HTon_40P_256Gmem_with_csv_2020.02.06T15.26.14|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.25.txt b/server/bin/gold/test-7.25.txt index dda618254b..a9060ed1bc 100644 --- a/server/bin/gold/test-7.25.txt +++ b/server/bin/gold/test-7.25.txt @@ -2377,6 +2377,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3482 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3726 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 964 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -2488,6 +2490,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -2502,3 +2506,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|rhel8-1|uperf_rhel8.1_4.18.0-107.el8_snap4_25gb_virt_2019.06.21T01.28.57|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.26.txt b/server/bin/gold/test-7.26.txt index c570f16152..00ec530920 100644 --- a/server/bin/gold/test-7.26.txt +++ b/server/bin/gold/test-7.26.txt @@ -1135,6 +1135,8 @@ drwxrwxr-x - logs/pbench-audit-server -rw-rw-r-- 437 logs/pbench-audit-server/pbench-audit-server.log drwxrwxr-x - logs/pbench-index-re -rw-rw-r-- 4025 logs/pbench-index-re/pbench-index-re.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - pbench-move-results-receive drwxrwxr-x - pbench-move-results-receive/fs-version-002 drwxrwxr-x - quarantine @@ -1219,7 +1221,12 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index-re.indexing_tarballs process_tb -- pbench-index-re.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index-re.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index-re/pbench-index-re.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log ---- pbench-local/logs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|ctlrA|linpack_mock_2020.02.28T19.10.55|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.3.txt b/server/bin/gold/test-7.3.txt index b60bf37e65..938848d02f 100644 --- a/server/bin/gold/test-7.3.txt +++ b/server/bin/gold/test-7.3.txt @@ -234,6 +234,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 265 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 2296 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -309,6 +311,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 0 (skipped 1) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -323,3 +327,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|alphaville|test_7.3_2015.09.21T15.31.08|UNPACKED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.4.txt b/server/bin/gold/test-7.4.txt index 537da1c143..43b8451af1 100644 --- a/server/bin/gold/test-7.4.txt +++ b/server/bin/gold/test-7.4.txt @@ -474,6 +474,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3375 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3616 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -585,6 +587,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -599,3 +603,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|alphaville|test_7.4_2015.09.21T15.31.08|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.5.txt b/server/bin/gold/test-7.5.txt index e33b3af013..ce6eb3976e 100644 --- a/server/bin/gold/test-7.5.txt +++ b/server/bin/gold/test-7.5.txt @@ -474,6 +474,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3375 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3616 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -585,6 +587,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -599,3 +603,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|alphaville|test_7.5_2015.09.21T15.31.08|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.6.txt b/server/bin/gold/test-7.6.txt index f9c5bcca1f..966313ad5e 100644 --- a/server/bin/gold/test-7.6.txt +++ b/server/bin/gold/test-7.6.txt @@ -475,6 +475,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3375 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3616 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -586,6 +588,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -600,3 +604,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|alphaville|test_7.6_2015.09.21T15.31.08|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.7.txt b/server/bin/gold/test-7.7.txt index 7a5cd4fa0e..f65d8f87bd 100644 --- a/server/bin/gold/test-7.7.txt +++ b/server/bin/gold/test-7.7.txt @@ -475,6 +475,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 3375 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3616 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -586,6 +588,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -600,3 +604,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|alphaville|test_7.7_2015.09.21T15.31.08|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.8.txt b/server/bin/gold/test-7.8.txt index 5ba9796ebf..b28b547ade 100644 --- a/server/bin/gold/test-7.8.txt +++ b/server/bin/gold/test-7.8.txt @@ -9370,6 +9370,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 30142 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3622 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 860 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9589,6 +9591,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9603,3 +9607,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|master_40gb|uperf__2016-10-06_16:34:03|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/gold/test-7.9.txt b/server/bin/gold/test-7.9.txt index 89f14276ff..710049537e 100644 --- a/server/bin/gold/test-7.9.txt +++ b/server/bin/gold/test-7.9.txt @@ -9428,6 +9428,8 @@ drwxrwxr-x - logs/pbench-index drwxrwxr-x - logs/pbench-index-tool-data -rw-rw-r-- 6911 logs/pbench-index-tool-data/pbench-index-tool-data.log -rw-rw-r-- 3666 logs/pbench-index/pbench-index.log +drwxrwxr-x - logs/pbench-state-manager +-rw-rw-r-- 0 logs/pbench-state-manager/pbench-state-manager.log drwxrwxr-x - logs/pbench-unpack-tarballs -rw-rw-r-- 0 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.error -rw-rw-r-- 905 logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9553,6 +9555,8 @@ drwxrwxr-x - tmp 1970-01-01T00:00:42.000000 INFO pbench-index.indexing_tarballs process_tb -- pbench-index.run-1970-01-01T00:00:42-UTC: indexed 1 (skipped 0) results, 0 errors 1970-01-01T00:00:42.000000 DEBUG pbench-index.report post_status -- posted status (start ts: 1970-01-01T00:00:42-UTC, end ts: 1970-01-01T00:00:42-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0) ----- pbench-index/pbench-index.log ++++++ pbench-state-manager/pbench-state-manager.log +----- pbench-state-manager/pbench-state-manager.log +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.error ----- pbench-unpack-tarballs/pbench-unpack-tarballs.error +++++ pbench-unpack-tarballs/pbench-unpack-tarballs.log @@ -9567,3 +9571,6 @@ run-1970-01-01T00:00:42-UTC: Processed 1 tarballs ++++ pbench-satellite-local/logs ---- pbench-satellite-local/logs --- pbench log file contents ++++ SqliteDB Datasets +public|dhcp31-144|pbench-user-benchmark__2017-04-21_20:38:16|INDEXED| +--- SqliteDB Datasets diff --git a/server/bin/pbench-backup-tarballs.py b/server/bin/pbench-backup-tarballs.py index 4246a4c204..f1df88cdde 100755 --- a/server/bin/pbench-backup-tarballs.py +++ b/server/bin/pbench-backup-tarballs.py @@ -16,6 +16,12 @@ from pbench.server.report import Report from pbench.server.s3backup import S3Config, Status, NoSuchKey from pbench.server.utils import rename_tb_link, quarantine +from pbench.server.database.models.tracker import ( + Dataset, + Metadata, + DatasetError, +) +from pbench.server.database.database import Database _NAME_ = "pbench-backup-tarballs" @@ -359,6 +365,14 @@ def backup_data(lb_obj, s3_obj, config, logger): resultname = tar.name controller_path = tar.parent controller = controller_path.name + try: + # This tool can't see a dataset until it's been prepared either + # by server PUT or by pbench-server-prep-shim-002.py; in either + # case, the Dataset object must already exist. + dataset = Dataset.attach(controller=controller, path=resultname) + except DatasetError as e: + logger.warning("Trouble tracking {}:{}: {}", controller, resultname, str(e)) + dataset = None # This will handle all the local backup related # operations and count the number of successes and failures. @@ -414,6 +428,9 @@ def backup_data(lb_obj, s3_obj, config, logger): # Do nothing when the backup fails, allowing us to retry on a # future pass. pass + + if dataset: + Metadata.create(dataset=dataset, key=Metadata.ARCHIVED, value="True") logger.debug("End backup of {}.", tar) return Results( @@ -444,6 +461,10 @@ def main(cfg_name): logger = get_pbench_logger(_NAME_, config) + # We're going to need the Postgres DB to track dataset state, so setup + # DB access. + Database.init_db(config, logger) + # Add a BACKUP and QDIR field to the config object config.BACKUP = config.conf.get("pbench-server", "pbench-backup-dir") config.QDIR = config.get("pbench-server", "pbench-quarantine-dir") diff --git a/server/bin/pbench-base.sh b/server/bin/pbench-base.sh index 06a7ad5799..f81a5d110e 100755 --- a/server/bin/pbench-base.sh +++ b/server/bin/pbench-base.sh @@ -135,6 +135,14 @@ function quarantine () { if [ ! -e $afile -a ! -L $afile ] ;then continue fi + # Create/update state record if we're not moving a .md5 file + if [ ${afile} != ${afile%.tar.xz} ] ;then + pbench-state-manager.py -c --path="${afile}" --state=quarantined + sts=${?} + if [[ ${sts} -ne 0 ]]; then + log_error "$TS: quarantine $afile: \"set state\" failed with status $sts" + fi + fi mv $afile $dest/ > /dev/null 2>&1 sts=$? if [ $sts -ne 0 ] ;then diff --git a/server/bin/pbench-index.py b/server/bin/pbench-index.py index 3792b51f24..13aa7b9afd 100755 --- a/server/bin/pbench-index.py +++ b/server/bin/pbench-index.py @@ -12,6 +12,7 @@ from pathlib import Path from argparse import ArgumentParser from configparser import Error as ConfigParserError +from pbench.server.database.database import Database from pbench.common.exceptions import ( BadConfig, @@ -147,6 +148,9 @@ def main(options, name): # Exit early if we encounter any errors. return res.value + # We're going to need the Postgres DB to track dataset state, so setup + # DB access. + Database.init_db(idxctx.config, idxctx.logger) idxctx.logger.debug("{}.{}: starting", name, idxctx.TS) index_obj = Index(name, options, idxctx, INCOMING_rp, ARCHIVE_rp, qdir) diff --git a/server/bin/pbench-reindex.py b/server/bin/pbench-reindex.py index 11601ec9d9..ce3b3b74d1 100755 --- a/server/bin/pbench-reindex.py +++ b/server/bin/pbench-reindex.py @@ -25,6 +25,13 @@ from pbench import BadConfig import pbench.server from pbench.server import PbenchServerConfig +from pbench.server.database.models.tracker import ( + Dataset, + Metadata, + DatasetError, +) +from pbench.server.database.database import Database +from pbench.common.logger import get_pbench_logger _NAME_ = "pbench-reindex" @@ -91,6 +98,11 @@ def reindex(controller_name, tb_name, archive_p, incoming_p, dry_run=False): try: if not dry_run: paths[0].rename(newpath) + ds = Dataset.attach(controller=controller_name, path=tb_name) + Metadata.create(dataset=ds, key=Metadata.REINDEX, value="True") + except DatasetError as exc: + msg = f"WARNING: unable to set REINDEX metadata for {controller_name}:{tb_name}: {str(exc)}" + res = "error" except Exception as exc: msg = ( f"WARNING: failed to rename symlink '{paths[0]}' to" @@ -168,6 +180,12 @@ def main(options): print(f"{_NAME_}: {e}", file=sys.stderr) return 2 + logger = get_pbench_logger(_NAME_, config) + + # We're going to need the Postgres DB to track dataset state, so setup + # DB access. + Database.init_db(config, logger) + archive_p = config.ARCHIVE try: diff --git a/server/bin/pbench-server-prep-shim-002.py b/server/bin/pbench-server-prep-shim-002.py index e37b343b96..d48c0dc788 100755 --- a/server/bin/pbench-server-prep-shim-002.py +++ b/server/bin/pbench-server-prep-shim-002.py @@ -23,6 +23,8 @@ from pbench.server import PbenchServerConfig from pbench.server.report import Report from pbench.server.utils import quarantine +from pbench.server.database.models.tracker import Dataset, States, DatasetError +from pbench.server.database.database import Database _NAME_ = "pbench-server-prep-shim-002" @@ -131,6 +133,27 @@ def process_tb(config, logger, receive_dir, qdir_md5, duplicates, errors): controller = tbdir.name dest = archive / controller + # Create a new dataset tracker in UPLOADING state, and add it to the + # database. + # + # NOTE: Technically, this particular workflow has no "UPLOADING" as + # the `pbench-server-prep-shim-002` command isn't invoked until the + # tarball and MD5 has been entirely uploaded by the agent via `ssh`; + # this method however can't be supported once we have authorized user + # ownership, and the model fits the server `PUT` method where an + # unexpected termination could leave a tarball in "Uploading" state. + try: + dataset = Dataset.create(controller=controller, path=resultname) + except DatasetError as e: + logger.error( + "Unable to create dataset {}>{}: {}", controller, resultname, str(e) + ) + # TODO: Should we quarantine over this? Note it's not quite + # straightforward, as quarantine() expects that the Dataset has + # been created, so we'll get a cascade failure. Since prep-shim's + # days are numbered, I'm inclined not to worry about it here. + dataset = None + if all([(dest / resultname).is_file(), (dest / tbmd5.name).is_file()]): logger.error("{}: Duplicate: {} duplicate name", config.TS, tb) quarantine((duplicates / controller), logger, tb, tbmd5) @@ -152,6 +175,15 @@ def process_tb(config, logger, receive_dir, qdir_md5, duplicates, errors): nquarantined += 1 continue + if dataset: + try: + dataset.md5 = archive_md5_hex_value + dataset.update() + except DatasetError as e: + logger.warn( + "Unable to update dataset {} with md5: {}", str(dataset), str(e) + ) + # make the destination directory and its TODO subdir if necessary. try: os.makedirs(dest / "TODO") @@ -236,6 +268,12 @@ def process_tb(config, logger, receive_dir, qdir_md5, duplicates, errors): ntbs += 1 + try: + if dataset: + dataset.advance(States.UPLOADED) + except Exception: + logger.exception("Unable to finalize {}", dataset) + nstatus = f"{nstatus}{config.TS}: processed {tb}\n" logger.info(f"{tb.name}: OK") @@ -267,6 +305,10 @@ def main(cfg_name): logger = get_pbench_logger(_NAME_, config) + # We're going to need the Postgres DB to track dataset state, so setup + # DB access. + Database.init_db(config, logger) + qdir, receive_dir = fetch_config_val(config, logger) if qdir is None and receive_dir is None: diff --git a/server/bin/pbench-state-manager b/server/bin/pbench-state-manager new file mode 120000 index 0000000000..debc8137d9 --- /dev/null +++ b/server/bin/pbench-state-manager @@ -0,0 +1 @@ +pbench-state-manager.py \ No newline at end of file diff --git a/server/bin/pbench-state-manager.py b/server/bin/pbench-state-manager.py new file mode 100755 index 0000000000..b9abbdbced --- /dev/null +++ b/server/bin/pbench-state-manager.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- + +"""Command line interface to the Dataset state mechanism. + +This serves two purposes: + +1. It allows access to the state tracker for bash scripts; +2. It's used to pre-set database state for the `gold` unit tests + +Therefore, while this can eventually be removed, we need to resolve both +of those requirements first. +""" + +import sys +import os + +from argparse import ArgumentParser + +from pbench import BadConfig +from pbench.server import PbenchServerConfig +from pbench.server.database.models.tracker import Dataset, States +from pbench.server.database.database import Database +from pbench.common.logger import get_pbench_logger + + +_NAME_ = "pbench-state-manager" + + +def main(options): + try: + if not options.cfg_name: + print( + f"{_NAME_}: ERROR: No config file specified; set" + " _PBENCH_SERVER_CONFIG env variable", + file=sys.stderr, + ) + return 1 + + try: + config = PbenchServerConfig(options.cfg_name) + except BadConfig as e: + print(f"{_NAME_}: {e}", file=sys.stderr) + return 2 + + logger = get_pbench_logger(_NAME_, config) + + # We're going to need the Postgres DB to track dataset state, so setup + # DB access. + Database.init_db(config, logger) + + args = {} + if options.controller: + args["controller"] = options.controller + if options.path: + args["path"] = options.path + if options.name: + args["name"] = options.name + if options.state: + try: + new_state = States[options.state.upper()] + except KeyError: + print( + f"{_NAME_}: Specified string '{options.state}' is not a Pbench dataset state", + file=sys.stderr, + ) + return 1 + args["state"] = new_state + + if "path" not in args and ("controller" not in args or "name" not in args): + print( + f"{_NAME_}: Either --path or both --controller and --name must be specified", + file=sys.stderr, + ) + return 1 + + # Either create a new dataset or attach to an existing dataset + doit = Dataset.create if options.create else Dataset.attach + + # Find the specified dataset, and update the state + doit(**args) + except Exception as e: + # Stringify any exception and report it; then fail + print(f"{_NAME_}: {e}", file=sys.stderr) + return 1 + else: + return 0 + + +if __name__ == "__main__": + parser = ArgumentParser(f"Usage: {_NAME_} [--config ]") + parser.add_argument( + "-C", + "--config", + dest="cfg_name", + default=os.environ.get("_PBENCH_SERVER_CONFIG"), + help="Specify config file", + ) + parser.add_argument( + "-c", + "--create", + dest="create", + action="store_true", + help="Create dataset instead of attaching to an existing dataset.", + ) + parser.add_argument( + "--path", + dest="path", + help="Specify a tarball filename (from which controller and name will be derived)", + ) + parser.add_argument( + "--controller", + dest="controller", + help="Specify controller name (agent host name)", + ) + parser.add_argument( + "--name", dest="name", help="Specify dataset name", + ) + parser.add_argument( + "--state", dest="state", help="Specify desired dataset state", + ) + parsed = parser.parse_args() + status = main(parsed) + sys.exit(status) diff --git a/server/bin/pbench-sync-satellite.sh b/server/bin/pbench-sync-satellite.sh index ea44a4820d..b7ac997035 100755 --- a/server/bin/pbench-sync-satellite.sh +++ b/server/bin/pbench-sync-satellite.sh @@ -211,6 +211,13 @@ for host in $hosts ;do log_error "Failed to create the symlink for md5 check passed tarballs: ln -sf $PWD/$x SATELLITE-MD5-PASSED/$x" "${mail_content}" continue fi + pbench-state-manager.py --create --controller="${remote_prefix}::${host}" --path="${x}" --state=uploaded + status=$? + if [[ $status != 0 ]] ;then + nerrs=$nerrs+1 + log_error "Failed to create dataset for $x" "${mail_content}" + continue + fi done fi grep 'FAILED' $logdir/$host/md5-checks.log > $logdir/$host/fail-checks.log diff --git a/server/bin/pbench-unpack-tarballs.sh b/server/bin/pbench-unpack-tarballs.sh index 73006fab60..17097e3c4b 100755 --- a/server/bin/pbench-unpack-tarballs.sh +++ b/server/bin/pbench-unpack-tarballs.sh @@ -206,6 +206,18 @@ function do_work() { if [[ ${SECONDS} -ge ${max_seconds} ]]; then break; fi continue fi + + # Record that we're currently unpacking this dataset + pbench-state-manager.py --controller="${hostname}" --name="${resultname}" --state=unpacking + status=${?} + if [[ ${status} -ne 0 ]]; then + log_error "${TS}: code ${status}: pbench-state-manager.py ${hostname} ${resultname} unpacking" "${mail_content}" + nerrs=${nerrs}+1 + move_symlink ${hostname} ${resultname} ${linksrc} ${linkerr} || doexit "Error handling failed for state update" + if [[ ${SECONDS} -ge ${max_seconds} ]]; then break; fi + continue + fi + let start_time=$(timestamp-seconds-since-epoch) tar --extract --no-same-owner --touch --delay-directory-restore --file="${result}" --force-local --directory="${incoming}.unpack" status=${?} @@ -333,6 +345,19 @@ function do_work() { fi fi + # Finalize the state transition to UNPACKED + pbench-state-manager.py --controller="${hostname}" --name="${resultname}" --state=unpacked + status=${?} + if [[ ${status} -ne 0 ]]; then + log_error "${TS}: code ${status}: pbench-state-manager.py ${hostname} ${resultname} unpacked" "${mail_content}" + nerrs=${nerrs}+1 + rm -rf ${incoming} + rm -f ${RESULTS}/${hostname}/${prefix}${resultname} + move_symlink ${hostname} ${resultname} ${linksrc} ${linkerr} || doexit "Error handling failed for state finalization" + if [[ ${SECONDS} -ge ${max_seconds} ]]; then break; fi + continue + fi + move_symlink ${hostname} ${resultname} ${linksrc} ${linkdest} status=${?} if [[ ${status} -ne 0 ]]; then diff --git a/server/bin/state/test-16.db b/server/bin/state/test-16.db new file mode 100644 index 0000000000..77e2250891 --- /dev/null +++ b/server/bin/state/test-16.db @@ -0,0 +1,3 @@ +controller00 benchmark-result-large_1970-01-01T00:00:00 uploaded +controller01 benchmark-result-medium_1970-01-01T00:00:00 uploaded +controller02 benchmark-result-small_1970-01-01T00:00:00 uploaded diff --git a/server/bin/state/test-17.db b/server/bin/state/test-17.db new file mode 100644 index 0000000000..3362750aaf --- /dev/null +++ b/server/bin/state/test-17.db @@ -0,0 +1,3 @@ +controller01 benchmark-result-medium_1970-01-01T00:00:00 uploaded +controller00 benchmark-result-large_1970-01-01T00:00:00 uploaded +controller02 benchmark-result-small_1970-01-01T00:00:00 uploaded diff --git a/server/bin/state/test-29.2.db b/server/bin/state/test-29.2.db new file mode 100644 index 0000000000..9976cfb6f6 --- /dev/null +++ b/server/bin/state/test-29.2.db @@ -0,0 +1 @@ +ctlr-in-range tarball_re-index_1970.02.14T12.42.42 indexed diff --git a/server/bin/state/test-6.10.db b/server/bin/state/test-6.10.db new file mode 100644 index 0000000000..64b01079fa --- /dev/null +++ b/server/bin/state/test-6.10.db @@ -0,0 +1 @@ +foo fio__2016-08-18_15:47:09 uploaded diff --git a/server/bin/state/test-6.11.db b/server/bin/state/test-6.11.db new file mode 100644 index 0000000000..2fb64018cf --- /dev/null +++ b/server/bin/state/test-6.11.db @@ -0,0 +1 @@ +foo fio__2016-08-18_15:47:09 indexing diff --git a/server/bin/state/test-6.12.db b/server/bin/state/test-6.12.db new file mode 100644 index 0000000000..715dbe54ea --- /dev/null +++ b/server/bin/state/test-6.12.db @@ -0,0 +1 @@ +foo fio__2016-08-18_15:47:09 indexed diff --git a/server/bin/state/test-6.13.db b/server/bin/state/test-6.13.db new file mode 100644 index 0000000000..79b86251d6 --- /dev/null +++ b/server/bin/state/test-6.13.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 unpacked +foo pbench-user-benchmark__2016-08-24_21:32:01 unpacked diff --git a/server/bin/state/test-6.14.db b/server/bin/state/test-6.14.db new file mode 100644 index 0000000000..79b86251d6 --- /dev/null +++ b/server/bin/state/test-6.14.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 unpacked +foo pbench-user-benchmark__2016-08-24_21:32:01 unpacked diff --git a/server/bin/state/test-6.15.db b/server/bin/state/test-6.15.db new file mode 100644 index 0000000000..fc719e8007 --- /dev/null +++ b/server/bin/state/test-6.15.db @@ -0,0 +1 @@ +foo fio__2016-08-18_15:47:09 unpacked diff --git a/server/bin/state/test-6.16.db b/server/bin/state/test-6.16.db new file mode 100644 index 0000000000..5f3529df2d --- /dev/null +++ b/server/bin/state/test-6.16.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 unpacked +foo client_error_other unpacked diff --git a/server/bin/state/test-6.17.db b/server/bin/state/test-6.17.db new file mode 100644 index 0000000000..87394186d1 --- /dev/null +++ b/server/bin/state/test-6.17.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 unpacked +foo get_object_exception unpacked diff --git a/server/bin/state/test-6.4.db b/server/bin/state/test-6.4.db new file mode 100644 index 0000000000..ede149f735 --- /dev/null +++ b/server/bin/state/test-6.4.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 uploaded +foo pbench-user-benchmark__2016-08-24_21:32:01 uploaded diff --git a/server/bin/state/test-6.5.db b/server/bin/state/test-6.5.db new file mode 100644 index 0000000000..5d2be3020a --- /dev/null +++ b/server/bin/state/test-6.5.db @@ -0,0 +1,2 @@ +foo pbench-user-benchmark__2016-08-24_21:32:01 unpacked +foo fio__2016-08-18_15:47:09 unpacked diff --git a/server/bin/state/test-6.6.db b/server/bin/state/test-6.6.db new file mode 100644 index 0000000000..5d2be3020a --- /dev/null +++ b/server/bin/state/test-6.6.db @@ -0,0 +1,2 @@ +foo pbench-user-benchmark__2016-08-24_21:32:01 unpacked +foo fio__2016-08-18_15:47:09 unpacked diff --git a/server/bin/state/test-6.7.db b/server/bin/state/test-6.7.db new file mode 100644 index 0000000000..79b86251d6 --- /dev/null +++ b/server/bin/state/test-6.7.db @@ -0,0 +1,2 @@ +foo fio__2016-08-18_15:47:09 unpacked +foo pbench-user-benchmark__2016-08-24_21:32:01 unpacked diff --git a/server/bin/state/test-6.9.db b/server/bin/state/test-6.9.db new file mode 100644 index 0000000000..fc719e8007 --- /dev/null +++ b/server/bin/state/test-6.9.db @@ -0,0 +1 @@ +foo fio__2016-08-18_15:47:09 unpacked diff --git a/server/bin/state/test-7.1.db b/server/bin/state/test-7.1.db new file mode 100644 index 0000000000..8a06b2c316 --- /dev/null +++ b/server/bin/state/test-7.1.db @@ -0,0 +1 @@ +controller test_7.1_1970.01.01T00.00.00 uploaded diff --git a/server/bin/state/test-7.10.db b/server/bin/state/test-7.10.db new file mode 100644 index 0000000000..79f657dcfe --- /dev/null +++ b/server/bin/state/test-7.10.db @@ -0,0 +1 @@ +dhcp31-44 uperf_uperftest_2018.02.02T20.58.00 uploaded diff --git a/server/bin/state/test-7.11.db b/server/bin/state/test-7.11.db new file mode 100644 index 0000000000..0ac05bad99 --- /dev/null +++ b/server/bin/state/test-7.11.db @@ -0,0 +1 @@ +dhcp31-44 fio_rw_2018.02.01T22.40.57 uploaded diff --git a/server/bin/state/test-7.12.db b/server/bin/state/test-7.12.db new file mode 100644 index 0000000000..a58c91b77a --- /dev/null +++ b/server/bin/state/test-7.12.db @@ -0,0 +1 @@ +EC2::ip-172-31-52-154 pbench-user-benchmark__2018.02.05T20.35.36 uploaded diff --git a/server/bin/state/test-7.13.db b/server/bin/state/test-7.13.db new file mode 100644 index 0000000000..7e389987ea --- /dev/null +++ b/server/bin/state/test-7.13.db @@ -0,0 +1 @@ +b03-h01-1029p pbench-user-benchmark_mbruzek-test-2_2018.04.10T19.01.19 uploaded diff --git a/server/bin/state/test-7.14.db b/server/bin/state/test-7.14.db new file mode 100644 index 0000000000..7e389987ea --- /dev/null +++ b/server/bin/state/test-7.14.db @@ -0,0 +1 @@ +b03-h01-1029p pbench-user-benchmark_mbruzek-test-2_2018.04.10T19.01.19 uploaded diff --git a/server/bin/state/test-7.15.db b/server/bin/state/test-7.15.db new file mode 100644 index 0000000000..2b86ec8780 --- /dev/null +++ b/server/bin/state/test-7.15.db @@ -0,0 +1 @@ +master_40gb uperf__2016-10-06_16:34:03 uploaded diff --git a/server/bin/state/test-7.16.db b/server/bin/state/test-7.16.db new file mode 100644 index 0000000000..006aad3a84 --- /dev/null +++ b/server/bin/state/test-7.16.db @@ -0,0 +1 @@ +rhel8-4 uperf_rhel8_4.18.0-18.el8_40gb_pass_2018.10.04T06.53.43 uploaded diff --git a/server/bin/state/test-7.17.db b/server/bin/state/test-7.17.db new file mode 100644 index 0000000000..c4d91e3214 --- /dev/null +++ b/server/bin/state/test-7.17.db @@ -0,0 +1 @@ +ansible-host pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18 uploaded diff --git a/server/bin/state/test-7.18.db b/server/bin/state/test-7.18.db new file mode 100644 index 0000000000..fc6dc0d424 --- /dev/null +++ b/server/bin/state/test-7.18.db @@ -0,0 +1,2 @@ +bad-controller test_7.18_2018.02.05T15.31.08 uploaded +bad-controller pbench-user-benchmark__2018.02.05T20.35.36 uploaded diff --git a/server/bin/state/test-7.19.db b/server/bin/state/test-7.19.db new file mode 100644 index 0000000000..3605b01760 --- /dev/null +++ b/server/bin/state/test-7.19.db @@ -0,0 +1 @@ +perf122 trafficgen_basic-forwarding-example_tg:trex-profile_pf:forwarding_test.json_ml:5_tt:bs__2019-08-27T14:58:38 uploaded diff --git a/server/bin/state/test-7.2.0.db b/server/bin/state/test-7.2.0.db new file mode 100644 index 0000000000..60afa47e94 --- /dev/null +++ b/server/bin/state/test-7.2.0.db @@ -0,0 +1 @@ +controller test_7.2.0_1970.01.01T00.42.00 uploaded diff --git a/server/bin/state/test-7.2.1.db b/server/bin/state/test-7.2.1.db new file mode 100644 index 0000000000..3267d3c32d --- /dev/null +++ b/server/bin/state/test-7.2.1.db @@ -0,0 +1 @@ +controller uperf__2016-10-06_16:34:03 uploaded diff --git a/server/bin/state/test-7.20.db b/server/bin/state/test-7.20.db new file mode 100644 index 0000000000..cd1f44c43b --- /dev/null +++ b/server/bin/state/test-7.20.db @@ -0,0 +1 @@ +ctlrA fio_mock_2020.02.27T22.16.14 uploaded diff --git a/server/bin/state/test-7.21.db b/server/bin/state/test-7.21.db new file mode 100644 index 0000000000..91752a747e --- /dev/null +++ b/server/bin/state/test-7.21.db @@ -0,0 +1,2 @@ +ctlrA trafficgen_mock_2020.02.28T20.04.29 uploaded +ctlrA trafficgen_mock_2020.02.28T19.49.39 uploaded diff --git a/server/bin/state/test-7.22.db b/server/bin/state/test-7.22.db new file mode 100644 index 0000000000..37820d3ccf --- /dev/null +++ b/server/bin/state/test-7.22.db @@ -0,0 +1 @@ +ctlrA linpack_mock_2020.02.28T19.10.55 uploaded diff --git a/server/bin/state/test-7.23.db b/server/bin/state/test-7.23.db new file mode 100644 index 0000000000..cf789d7e19 --- /dev/null +++ b/server/bin/state/test-7.23.db @@ -0,0 +1 @@ +ctlrA fio_mock_2020.01.19T00.18.06 uploaded diff --git a/server/bin/state/test-7.24.db b/server/bin/state/test-7.24.db new file mode 100644 index 0000000000..61135bce1d --- /dev/null +++ b/server/bin/state/test-7.24.db @@ -0,0 +1,2 @@ +ctlrA pbench-user-benchmark_Maridb_tuned_TP_HTon_40P_256Gmem_with_csv_2020.02.06T15.26.14 uploaded +ctlrA pbench-user-benchmark_1000_80_latency_perf_2021.01.28T17.22.57 uploaded diff --git a/server/bin/state/test-7.25.db b/server/bin/state/test-7.25.db new file mode 100644 index 0000000000..2deba2995a --- /dev/null +++ b/server/bin/state/test-7.25.db @@ -0,0 +1 @@ +rhel8-1 uperf_rhel8.1_4.18.0-107.el8_snap4_25gb_virt_2019.06.21T01.28.57 uploaded diff --git a/server/bin/state/test-7.26.db b/server/bin/state/test-7.26.db new file mode 100644 index 0000000000..f37c042fed --- /dev/null +++ b/server/bin/state/test-7.26.db @@ -0,0 +1 @@ +ctlrA linpack_mock_2020.02.28T19.10.55 unpacked diff --git a/server/bin/state/test-7.3.db b/server/bin/state/test-7.3.db new file mode 100644 index 0000000000..38deb65516 --- /dev/null +++ b/server/bin/state/test-7.3.db @@ -0,0 +1 @@ +alphaville test_7.3_2015.09.21T15.31.08 uploaded diff --git a/server/bin/state/test-7.4.db b/server/bin/state/test-7.4.db new file mode 100644 index 0000000000..0cde2a31ec --- /dev/null +++ b/server/bin/state/test-7.4.db @@ -0,0 +1 @@ +alphaville test_7.4_2015.09.21T15.31.08 uploaded diff --git a/server/bin/state/test-7.5.db b/server/bin/state/test-7.5.db new file mode 100644 index 0000000000..08845e14e2 --- /dev/null +++ b/server/bin/state/test-7.5.db @@ -0,0 +1 @@ +alphaville test_7.5_2015.09.21T15.31.08 uploaded diff --git a/server/bin/state/test-7.6.db b/server/bin/state/test-7.6.db new file mode 100644 index 0000000000..264568cbf3 --- /dev/null +++ b/server/bin/state/test-7.6.db @@ -0,0 +1 @@ +alphaville test_7.6_2015.09.21T15.31.08 uploaded diff --git a/server/bin/state/test-7.7.db b/server/bin/state/test-7.7.db new file mode 100644 index 0000000000..ee4b8f18c9 --- /dev/null +++ b/server/bin/state/test-7.7.db @@ -0,0 +1 @@ +alphaville test_7.7_2015.09.21T15.31.08 uploaded diff --git a/server/bin/state/test-7.8.db b/server/bin/state/test-7.8.db new file mode 100644 index 0000000000..2b86ec8780 --- /dev/null +++ b/server/bin/state/test-7.8.db @@ -0,0 +1 @@ +master_40gb uperf__2016-10-06_16:34:03 uploaded diff --git a/server/bin/state/test-7.9.db b/server/bin/state/test-7.9.db new file mode 100644 index 0000000000..68664a7577 --- /dev/null +++ b/server/bin/state/test-7.9.db @@ -0,0 +1 @@ +dhcp31-144 pbench-user-benchmark__2017-04-21_20:38:16 uploaded diff --git a/server/bin/unittests b/server/bin/unittests index b5efd89e55..2d65e91d8a 100755 --- a/server/bin/unittests +++ b/server/bin/unittests @@ -88,7 +88,7 @@ function _run_activate { } function _run_allscripts { - # These three pull in new tar balls from move-results and remote + # The first two pull in new tar balls from move-results and remote # satellite pbench servers feeding them into the dispatch loop. _run pbench-server-prep-shim-002 # NOTE WELL: the "satellite-one" argument refers a configuration @@ -184,6 +184,9 @@ function _audit_server { } function _dump_logs { + local count + local db + local select # Dump the state of any generated script logs echo "+++ pbench log file contents" >> $_testout for dir in ${_testdir} ${_testdir_local} ${_testdir_sat} ${_testdir_sat_local} ;do @@ -217,6 +220,17 @@ function _dump_logs { _normalize_output $_testloggerpayload >> $_testout 2>&1 echo "--- $(basename $_testloggerpayload) file contents" >> $_testout fi + + # Be quiet if there's no database, or if the `dataset` table hasn't been + # created. + db=${_testroot}/test_db + select='select user,controller,name,state,(select " " || key || " = " || value from metadata where dataset_ref = dataset.id) from dataset order by user,controller,name asc' + sqlite3 ${db} "${select}" > ${_testdir_local}/db 2>/dev/null + if [ -s ${_testdir_local}/db ] ;then + echo "+++ SqliteDB Datasets" >> $_testout + _normalize_output ${_testdir_local}/db >> ${_testout} + echo "--- SqliteDB Datasets" >> $_testout + fi } function _verify_output { @@ -354,6 +368,8 @@ function _setup_state { printf "[DEFAULT]\n" >> ${_testtmp}/pbench-server.cfg printf "unittest-dir = ${_testroot}\n" >> ${_testtmp}/pbench-server.cfg cat ${_state_config} >> ${_testtmp}/pbench-server.cfg + printf "[Postgres]\n" >> ${_testtmp}/pbench-server.cfg + printf "db_uri = sqlite:///${_testroot}/test_db\n" >> ${_testtmp}/pbench-server.cfg # First we activate the main pbench server (not a satellite). echo "$_testopt/bin/pbench-server-config-activate ${_testtmp}/pbench-server.cfg" >> $_testactout @@ -436,6 +452,14 @@ function _setup_state { exit 1 fi fi + + # Setup per-test Dataset DB + _state_db=${_tdir}/state/${1}.db + if [ -f ${_state_db} ]; then + while read -r controller name state; do + pbench-state-manager --create --controller=${controller} --name=${name} --state=${state} + done < ${_state_db} + fi } function _run_test { @@ -841,6 +865,7 @@ function report_test_result { echo "${pass_or_fail} - ${testname} ($(cat ${_testbase}/${testname}/result.duration))" rm ${_testbase}/${testname}/result.txt rm -f ${_testbase}/${testname}/result.duration + rm -f ${_testbase}/${testname}/test_db rmdir ${_testbase}/${testname} res=0 else diff --git a/server/requirements.txt b/server/requirements.txt index 79c107e8d0..60560024a3 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -17,3 +17,4 @@ PyJwt psycopg2-binary requests sqlalchemy +sqlalchemy_utils