Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug 1910469 - Update sanitizeme script to truncate bugs_fulltext table to significantly reduce size of dump file #2287

Merged
merged 1 commit into from
Jul 31, 2024
Merged

Bug 1910469 - Update sanitizeme script to truncate bugs_fulltext table to significantly reduce size of dump file #2287

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions scripts/sanitizeme.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@
# a user who is not logged in.

my (
$dry_run, $from_cron, $keep_attachments, $keep_group_bugs,
$keep_groups, $execute, $keep_passwords, $keep_insider,
$trace, $enable_email
) = (0, 0, 0, '', 0, 0, 0, 0, 0, 0);
$dry_run, $from_cron, $keep_attachments, $keep_group_bugs,
$keep_groups, $execute, $keep_passwords, $keep_insider,
$trace, $enable_email, $keep_fulltext
) = (0, 0, 0, '', 0, 0, 0, 0, 0, 0, 0);
my $keep_group_bugs_sql = '';

my $syntax = <<EOF;
Expand All @@ -57,6 +57,7 @@
--keep-group-bugs disable removal of the specified groups and associated bugs
--keep-groups disable removal of group definitions
--enable-email do not disable email for all users
--keep-fulltext disable removal of fulltext data
--dry-run do not update the database, just output what will be deleted
--from-cron quite mode - suppress non-warning/error output
--trace output sql statements
Expand All @@ -72,6 +73,7 @@
"keep-groups" => \$keep_groups,
"trace" => \$trace,
"enable-email" => \$enable_email,
"keep-fulltext" => \$keep_fulltext,
) or die $syntax;
die "--execute switch required to perform database sanitization.\n\n$syntax"
unless $execute or $dry_run;
Expand Down Expand Up @@ -110,6 +112,7 @@
delete_user_request_log();
Bugzilla::Hook::process('db_sanitize');
disable_email_delivery() unless $enable_email;
delete_bugs_fulltext() unless $keep_fulltext;
delete_system_parameters();
delete_oauth_providers();
delete_bloomfilter();
Expand Down Expand Up @@ -171,16 +174,17 @@ sub delete_deleted_comments {
"SELECT comment_id FROM longdescs_tags WHERE tag='deleted'");
return unless @$comment_ids;
print "Deleting 'deleted' comments...\n";
my @bug_ids = uniq @{
$dbh->selectcol_arrayref(
"SELECT bug_id FROM longdescs WHERE comment_id IN ("
. join(',', @$comment_ids) . ")"
)
};
$dbh->do(
"DELETE FROM longdescs WHERE comment_id IN (" . join(',', @$comment_ids) . ")");
foreach my $bug_id (@bug_ids) {
Bugzilla::Bug->new($bug_id)->_sync_fulltext(update_comments => 1);

if ($keep_fulltext) {
my $bug_ids
= $dbh->selectcol_arrayref(
'SELECT DISTINCT bug_id FROM longdescs WHERE comment_id IN (' . join ',',
@$comment_ids . ')');
foreach my $bug_id (@{$bug_ids}) {
Bugzilla::Bug->new($bug_id)->_sync_fulltext(update_comments => 1);
}
}
}

Expand All @@ -193,7 +197,7 @@ sub delete_insider_comments {
"DELETE attach_data FROM attachments JOIN attach_data ON attachments.attach_id = attach_data.id WHERE attachments.isprivate = 1"
);
$dbh->do("DELETE FROM attachments WHERE isprivate = 1");
$dbh->do("UPDATE bugs_fulltext SET comments = comments_noprivate");
$dbh->do('UPDATE bugs_fulltext SET comments = comments_noprivate') if $keep_fulltext;
}

sub delete_security_groups {
Expand Down Expand Up @@ -284,6 +288,13 @@ sub disable_email_delivery {
$dbh->do("UPDATE flagtypes SET cc_list = NULL");
}

sub delete_bugs_fulltext {
# Delete data from the bugs_fulltext table as the data
# can be regenerated if needed
print "Deleting bugs fulltext data...\n";
$dbh->do('TRUNCATE TABLE bugs_fulltext');
}

sub delete_system_parameters {
print "Deleting system parameters...\n";
$dbh->do('DELETE FROM params');
Expand Down
Loading