diff --git a/lmfdb/backend/base.py b/lmfdb/backend/base.py index 2aa903f087..c7ab35a752 100644 --- a/lmfdb/backend/base.py +++ b/lmfdb/backend/base.py @@ -122,8 +122,6 @@ def jsonb_idx(cols, cols_type): "total", "important", "include_nones", - "table_description", - "col_description", ) _meta_tables_cols_notrequired = ( "count_cutoff", @@ -131,9 +129,7 @@ def jsonb_idx(cols, cols_type): "total", "important", "include_nones", - "table_description", - "col_description", -) # defaults: 1000, true, 0, false, false, "", {} +) # defaults: 1000, true, 0, false, false _meta_tables_types = dict(zip(_meta_tables_cols, ( "text", "jsonb", @@ -146,8 +142,6 @@ def jsonb_idx(cols, cols_type): "bigint", "boolean", "boolean", - "text", - "jsonb", ))) _meta_tables_jsonb_idx = jsonb_idx(_meta_tables_cols, _meta_tables_types) @@ -222,12 +216,13 @@ def __init__(self, loggername, db): self.slow_cutoff = logging_options["slowcutoff"] self.logger = l = logging.getLogger(loggername) l.propagate = False + # we only want 2 handlers + l.handlers = [] l.setLevel(logging_options.get('loglevel', logging.INFO)) - fhandler = logging.FileHandler(logging_options["slowlogfile"]) formatter = logging.Formatter("%(asctime)s - %(message)s") - filt = QueryLogFilter() + fhandler = logging.FileHandler(logging_options["slowlogfile"]) fhandler.setFormatter(formatter) - fhandler.addFilter(filt) + fhandler.addFilter(QueryLogFilter()) l.addHandler(fhandler) shandler = logging.StreamHandler() shandler.setFormatter(formatter) diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index 8148081bc5..8b4f04ec60 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -44,11 +44,11 @@ def setup_connection(conn): register_json(conn, loads=Json.loads) try: from sage.all import Integer, RealNumber + from .encoding import RealEncoder, LmfdbRealLiteral except ImportError: pass else: register_adapter(Integer, AsIs) - from .encoding import RealEncoder, LmfdbRealLiteral register_adapter(RealNumber, RealEncoder) register_adapter(LmfdbRealLiteral, RealEncoder) @@ -489,13 +489,13 @@ def _create_meta_tables_hist(self): "(name text, sort jsonb, count_cutoff smallint DEFAULT 1000, " "id_ordered boolean, out_of_order boolean, has_extras boolean, " "stats_valid boolean DEFAULT true, label_col text, total bigint, " - "include_nones boolean, table_description text, col_description jsonb, version integer)" + "include_nones boolean, version integer)" )) version = 0 # copy data from meta_tables rows = self._execute(SQL( - "SELECT name, sort, id_ordered, out_of_order, has_extras, label_col, total, include_nones, table_description, col_description FROM meta_tables " + "SELECT name, sort, id_ordered, out_of_order, has_extras, label_col, total, include_nones FROM meta_tables " )) for row in rows: @@ -503,8 +503,8 @@ def _create_meta_tables_hist(self): SQL( "INSERT INTO meta_tables_hist " "(name, sort, id_ordered, out_of_order, has_extras, label_col, " - "total, include_nones, table_description, col_description, version) " - "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + "total, include_nones, version) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)" ), row + (version,), ) @@ -778,8 +778,8 @@ def process_columns(coldict, colorder): # FIXME use global constants ? inserter = SQL( "INSERT INTO meta_tables " - "(name, sort, id_ordered, out_of_order, has_extras, label_col, table_description, col_description) " - "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)" + "(name, sort, id_ordered, out_of_order, has_extras, label_col) " + "VALUES (%s, %s, %s, %s, %s, %s)" ) self._execute( inserter, @@ -790,11 +790,9 @@ def process_columns(coldict, colorder): not id_ordered, extra_columns is not None, label_col, - table_description, - Json(col_description), ], ) - self.__dict__[name] = self._search_table_class_( + new_table = self._search_table_class_( self, name, label_col, @@ -804,6 +802,9 @@ def process_columns(coldict, colorder): has_extras=(extra_columns is not None), total=0, ) + new_table.description(table_description) + new_table.column_description(description=col_description) + self.__dict__[name] = new_table self.tablenames.append(name) self.tablenames.sort() self.log_db_change( @@ -1117,8 +1118,6 @@ def reload_all( if len(rows) != 1: raise RuntimeError("Expected only one row in {0}") meta = dict(zip(_meta_tables_cols, rows[0])) - import ast - meta["col_description"] = ast.literal_eval(meta["col_description"]) assert meta["name"] == tablename with search_table_file.open("r") as F: @@ -1141,7 +1140,7 @@ def reload_all( extra_columns[typ].append(name) # the rest of the meta arguments will be replaced on the reload_all # We use force_description=False so that beta and prod can be out-of-sync with respect to columns and/or descriptions - self.create_table(tablename, search_columns, None, table_description=meta["table_description"], col_description=meta["col_description"], extra_columns=extra_columns, force_description=False) + self.create_table(tablename, search_columns, None, extra_columns=extra_columns, force_description=False) for tablename in self.tablenames: included = [] diff --git a/lmfdb/backend/statstable.py b/lmfdb/backend/statstable.py index e46b705e66..e0b2cee11a 100644 --- a/lmfdb/backend/statstable.py +++ b/lmfdb/backend/statstable.py @@ -1581,6 +1581,7 @@ def refresh_stats(self, total=True, reset_None_to_1=False, suffix=""): if total: # Refresh total in meta_tables self.total = self._slow_count({}, suffix=suffix, extra=False) + self.refresh_null_counts(suffix=suffix) self.logger.info("Refreshed statistics in %.3f secs" % (time.time() - t0)) def status(self, reset_None_to_1=False): diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index 1d07f5f54b..00427c6ef4 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -1171,7 +1171,6 @@ def drop_tmp(): if not self._table_exists(table + "_tmp"): self._clone(table, table + "_tmp") self.stats.refresh_stats(suffix=suffix) - self.stats.refresh_null_counts(suffix=suffix) if not inplace: swapped_tables = ( [self.search_table] @@ -1786,7 +1785,7 @@ def reload( if table not in tables: tables.append(table) - if countsfile: + if self.stats.counts in tables: # create index on counts table self._create_counts_indexes(suffix=suffix) @@ -2217,73 +2216,33 @@ def get_label(self): def description(self, table_description=None): """ - Return or set the description string for this table in meta_tables + This stub defines the API for getting and setting the table description. + In the LMFDB, this is implemented using the knowl table, but we do nothing by default. INPUT: - - ``table_description`` -- if provided, set the description to this value. If not, return the current description. + - ``table_description`` -- if provided, set the description to this value. + If not, return the current description. """ - if table_description is None: - selecter = SQL("SELECT table_description FROM meta_tables WHERE name = %s") - desc = list(self._execute(selecter, [self.search_table])) - if desc and desc[0]: - return desc[0] - else: - return "(table description not yet updated on this server)" - else: - assert isinstance(table_description, str) - modifier = SQL("UPDATE meta_tables SET table_description = %s WHERE name = %s") - self._execute(modifier, [table_description, self.search_table]) + pass def column_description(self, col=None, description=None, drop=False): """ - Set the description for a column in meta_tables. + This stub defines the API for getting, setting and deleting column descriptions. + In the LMFDB, this is implemented using the knowl table, but we do nothing by default. INPUT: - - ``col`` -- the name of the column. If None, ``description`` should be a dictionary with keys equal to the column names. + - ``col`` -- the name of the column. If None, ``description`` should be a dictionary + with keys equal to the column names. - - ``description`` -- if provided, set the column description to this value. If not, return the current description. + - ``description`` -- if provided, set the column description to this value. + If not, return the current description. - - ``drop`` -- if ``True``, delete the column from the description dictionary in preparation for dropping the column. + - ``drop`` -- if ``True``, delete the column from the description dictionary in + preparation for dropping the column. """ - allcols = self.search_cols + self.extra_cols - # Get the current column description - selecter = SQL("SELECT col_description FROM meta_tables WHERE name = %s") - cur = self._execute(selecter, [self.search_table]) - current = cur.fetchone()[0] - - if not drop and description is None: - # We want to allow the set of columns to be out of date temporarily, on prod for example - if col is None: - for col in allcols: - if col not in current: - current[col] = "(description not yet updated on this server)" - return current - return current.get(col, "(description not yet updated on this server)") - else: - if not (drop or col is None or col in allcols): - raise ValueError("%s is not a column of this table" % col) - if drop: - if col is None: - raise ValueError("Must specify column name to drop") - try: - del current[col] - except KeyError: - # column was already not present for some reason - return - elif col is None: - assert isinstance(description, dict) - for col in description: - if col not in allcols: - raise ValueError("%s is not a column of this table" % col) - assert isinstance(description[col], str) - current[col] = description[col] - else: - assert isinstance(description, str) - current[col] = description - modifier = SQL("UPDATE meta_tables SET col_description = %s WHERE name = %s") - self._execute(modifier, [Json(current), self.search_table]) + pass def add_column(self, name, datatype, description=None, extra=False, label=False, force_description=False): """ diff --git a/lmfdb/backend/utils.py b/lmfdb/backend/utils.py index 3022b1a26e..f7f90e6585 100644 --- a/lmfdb/backend/utils.py +++ b/lmfdb/backend/utils.py @@ -171,7 +171,7 @@ class QueryLogFilter(): """ def filter(self, record): - if record.pathname.startswith("db_backend.py"): + if record.pathname.endswith("base.py"): return 1 else: return 0 diff --git a/lmfdb/knowledge/knowl.py b/lmfdb/knowledge/knowl.py index cc3b771757..df5d190675 100644 --- a/lmfdb/knowledge/knowl.py +++ b/lmfdb/knowledge/knowl.py @@ -23,6 +23,7 @@ top_knowl_re = re.compile(r"(.*)\.top$") comment_knowl_re = re.compile(r"(.*)\.(\d+)\.comment$") coldesc_knowl_re = re.compile(r"columns.([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)") +tabledesc_knowl_re = re.compile(r"tables.([A-Za-z0-9_]+)") bottom_knowl_re = re.compile(r"(.*)\.bottom$") url_from_knowl = [ (re.compile(r'g2c\.(\d+\.[a-z]+\.\d+\.\d+)'), 'Genus2Curve/Q/{0}', 'Genus 2 curve {0}'), @@ -86,6 +87,9 @@ def extract_typ(kid): m = coldesc_knowl_re.match(kid) if m: return 2, m.group(1), m.group(2) + m = tabledesc_knowl_re.match(kid) + if m: + return 2, None, m.group(1) m = top_knowl_re.match(kid) if m: prelabel = m.group(1) @@ -289,8 +293,8 @@ def save(self, knowl, who, most_recent=None, minor=False): else: typ, source, name = extract_typ(knowl.id) links = extract_links(knowl.content) - if typ == 2: # column description - defines = [knowl.id.split(".")[-1]] + if typ == 2: # column or table description + defines = [name] else: defines = extract_defines(knowl.content) # id, authors, cat, content, last_author, timestamp, title, status, type, links, defines, source, source_name @@ -354,6 +358,31 @@ def drop_column(self, table, col): kwl = Knowl(kid, data=self.get_knowl(kid, beta=True)) self.delete(kwl) + def get_table_description(self, table): + fields = ['id'] + self._default_fields + selecter = SQL("SELECT {0} FROM (SELECT DISTINCT ON (id) {0} FROM kwl_knowls WHERE id = %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id").format(SQL(", ").join(map(Identifier, fields))) + rec = self._execute(selecter, [f"tables.{table}", 2, 0]).fetchone() + if rec: + return Knowl(rec[0], data=dict(zip(fields, rec))) + + def set_table_description(self, table, description): + uid = db.login() + kid = f"tables.{table}" + data = { + 'content': description, + 'defines': table, + } + kwl = Knowl(kid, data=data) + old = self.get_knowl(kid, beta=True) + if old is None: + old = {'authors': []} + self.save(kwl, uid, most_recent=old) + + def drop_table(self, table): + kid = f"tables.{table}" + kwl = Knowl(kid, data=self.get_knowl(kid, beta=True)) + self.delete(kwl) + def delete(self, knowl): """deletes this knowl from the db. This is effected by setting the status to -2 on all copies of the knowl""" updator = SQL("UPDATE kwl_knowls SET status=%s WHERE id=%s") @@ -815,11 +844,18 @@ def __init__(self, ID, template_kwargs=None, data=None, editing=False, showing=F if self.type == 2: pieces = ID.split(".") # Ignore the title passed in - self.title = f"Column {pieces[2]} of table {pieces[1]}" - if pieces[1] in db.tablenames: - self.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT") - else: - self.coltype = "DEFUNCT" + if len(pieces) == 3: + # Column + self.title = f"Column {pieces[2]} of table {pieces[1]}" + if pieces[1] in db.tablenames: + self.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT") + else: + self.coltype = "DEFUNCT" + elif len(pieces) == 2: + # Table + self.title = f"Table {pieces[1]}" + if pieces[1] not in db.tablenames: + self.title += " (DEFUNCT)" #self.reviewer = data.get('reviewer') # Not returned by get_knowl by default #self.review_timestamp = data.get('review_timestamp') # Not returned by get_knowl by default diff --git a/lmfdb/lmfdb_database.py b/lmfdb/lmfdb_database.py index a695ad7acb..3ffdd748b4 100644 --- a/lmfdb/lmfdb_database.py +++ b/lmfdb/lmfdb_database.py @@ -34,9 +34,23 @@ def __init__(self, *args, **kwds): PostgresSearchTable.__init__(self, *args, **kwds) self._verifier = None # set when importing lmfdb.verify + def description(self, table_description=None): + """ + We use knowls to implement the table description API. + """ + from lmfdb.knowledge.knowl import knowldb + if table_description is None: + current = knowldb.get_table_description(self.search_table) + if current: + return current.content + else: + return "(description not yet updated on this server)" + else: + knowldb.set_table_description(self.search_table, table_description) + def column_description(self, col=None, description=None, drop=False): """ - We use knowls to store column descriptions rather than meta_tables. + We use knowls to implement the column description API. """ from lmfdb.knowledge.knowl import knowldb allcols = self.search_cols + self.extra_cols @@ -460,5 +474,14 @@ def create_table(self, name, *args, **kwargs): kwargs["force_description"] = True return PostgresDatabase.create_table(self, name, *args, **kwargs) + @overrides(PostgresDatabase) + def drop_table(self, name, *args, **kwargs): + cols = self[name].search_cols + self[name].extra_cols + super().drop_table(name, *args, **kwargs) + from lmfdb.knowledge.knowl import knowldb + knowldb.drop_table(name) + for col in cols: + knowldb.drop_column(name, col) + print("Deleted table and column descriptions from knowl database") db = LMFDBDatabase()