diff --git a/CopyIndexes.mkdn b/CopyIndexes.mkdn index 2f5c62a..77bfae4 100644 --- a/CopyIndexes.mkdn +++ b/CopyIndexes.mkdn @@ -4,7 +4,7 @@ es-copy-index.pl - Copy an index from one cluster to another # VERSION -version 8.6 +version 8.7 # SYNOPSIS @@ -42,7 +42,6 @@ From App::ElasticSearch::Utilities: --index Index to run commands against --base For daily indexes, reference only those starting with "logstash" (same as --pattern logstash-* or logstash-DATE) - --datesep Date separator, default '.' also (--date-separator) --pattern Use a pattern to operate on the indexes --days If using a pattern or base, how many days back to go, default: 1 diff --git a/Maintenance.mkdn b/Maintenance.mkdn index 159eebb..c8ba7e2 100644 --- a/Maintenance.mkdn +++ b/Maintenance.mkdn @@ -4,7 +4,7 @@ es-daily-index-maintenance.pl - Run to prune old indexes and optimize existing # VERSION -version 8.6 +version 8.7 # SYNOPSIS @@ -46,7 +46,6 @@ From App::ElasticSearch::Utilities: --index Index to run commands against --base For daily indexes, reference only those starting with "logstash" (same as --pattern logstash-* or logstash-DATE) - --datesep Date separator, default '.' also (--date-separator) --pattern Use a pattern to operate on the indexes --days If using a pattern or base, how many days back to go, default: 1 diff --git a/README.mkdn b/README.mkdn index 4f75bde..5fd588d 100644 --- a/README.mkdn +++ b/README.mkdn @@ -4,7 +4,7 @@ App::ElasticSearch::Utilities - Utilities for Monitoring ElasticSearch # VERSION -version 8.6 +version 8.7 # SYNOPSIS @@ -285,7 +285,6 @@ From App::ElasticSearch::Utilities: --index Index to run commands against --base For daily indexes, reference only those starting with "logstash" (same as --pattern logstash-* or logstash-DATE) - --datesep Date separator, default '.' also (--date-separator) --pattern Use a pattern to operate on the indexes --days If using a pattern or base, how many days back to go, default: 1 diff --git a/Searching.mkdn b/Searching.mkdn index cb8478f..e13b0b8 100644 --- a/Searching.mkdn +++ b/Searching.mkdn @@ -4,7 +4,7 @@ es-search.pl - Provides a CLI for quick searches of data in ElasticSearch daily # VERSION -version 8.6 +version 8.7 # SYNOPSIS @@ -62,7 +62,6 @@ From App::ElasticSearch::Utilities: --index Index to run commands against --base For daily indexes, reference only those starting with "logstash" (same as --pattern logstash-* or logstash-DATE) - --datesep Date separator, default '.' also (--date-separator) --pattern Use a pattern to operate on the indexes --days If using a pattern or base, how many days back to go, default: 1 diff --git a/lib/App/ElasticSearch/Utilities.pm b/lib/App/ElasticSearch/Utilities.pm index 7ff0a4e..2ab16a7 100644 --- a/lib/App/ElasticSearch/Utilities.pm +++ b/lib/App/ElasticSearch/Utilities.pm @@ -101,7 +101,6 @@ From App::ElasticSearch::Utilities: --index Index to run commands against --base For daily indexes, reference only those starting with "logstash" (same as --pattern logstash-* or logstash-DATE) - --datesep Date separator, default '.' also (--date-separator) --pattern Use a pattern to operate on the indexes --days If using a pattern or base, how many days back to go, default: 1 @@ -249,9 +248,19 @@ The indexes are compared against this pattern. # Global Variables our %_GLOBALS = (); my %DEF = (); -my %PATTERN_REGEX = ( +my %PATTERN_REGEX = ( '*' => qr/.*/, ANY => qr/.*/, + DATE => qr/ + (? + (?\d{4}) # Extract 4 digits for the year + (?:(?[\-.]))? # Optionally, look for . - as a separator + (?\d{2}) # Two digits for the month + \g{datesep} # Whatever the date separator was in the previous match + (?\d{2}) # Two digits for the day + (?![a-zA-Z0-9]) # Zero width negative look ahead, not alphanumeric + ) + /x, ); my $PATTERN; @@ -283,7 +292,6 @@ my $PATTERN; base|index-basename=s days=i noop! - datesep|date-separator=s proto=s http-username=s password-exec=s @@ -390,10 +398,6 @@ sub es_utils_initialize { PATTERN => exists $opts->{pattern} ? $opts->{pattern} : '*', DAYS => exists $opts->{days} ? $opts->{days} : exists $_GLOBALS{days} ? $_GLOBALS{days} : 1, - DATESEP => exists $opts->{datesep} ? $opts->{datesep} - : exists $_GLOBALS{datesep} ? $_GLOBALS{datesep} - : exists $_GLOBALS{"date-separator"} ? $_GLOBALS{"date-separator"} - : '.', # HTTP Basic Authentication USERNAME => exists $opts->{'http-username'} ? $opts->{'http-username'} : exists $_GLOBALS{'http-username'} ? $_GLOBALS{'http-username'} @@ -425,20 +429,11 @@ sub es_utils_initialize { delete $ENV{$_} for qw(http_proxy HTTP_PROXY); } - # Setup Variables based on the config - %PATTERN_REGEX = ( - '*' => qr/.*/, - DATE => qr/\d{4}(?:\Q$DEF{DATESEP}\E)?\d{2}(?:\Q$DEF{DATESEP}\E)?\d{2}/, - ANY => qr/.*/, - ); - my @ordered = qw(* DATE ANY); - - if( index($DEF{DATESEP},'-') >= 0 ) { - output({stderr=>1,color=>'yellow'}, "=== Using a '-' as your date separator may cause problems with other utilities. ==="); - } # Build the Index Pattern $PATTERN = $DEF{PATTERN}; + + my @ordered = qw(* DATE ANY); foreach my $literal ( @ordered ) { $PATTERN =~ s/\Q$literal\E/$PATTERN_REGEX{$literal}/g; } @@ -1043,7 +1038,6 @@ sub es_indices { } } elsif( $args{check_dates} && defined $DEF{DAYS} ) { - my $days_old = es_index_days_old( $index ); if( !defined $days_old ) { debug({indent=>2,color=>'red'}, "! error locating date in string, skipping !"); @@ -1081,12 +1075,8 @@ sub es_index_strip_date { es_utils_initialize() unless keys %DEF; # Try the Date Pattern - if( $index =~ s/[-_]$PATTERN_REGEX{DATE}.*//o ) { - return $index; - } - # Fallback to matching thing-YYYY-MM-DD or thing-YYYY.MM.DD - elsif( $index =~ s/[-_]\d{4}([.-])\d{2}\g{1}\d{2}(?:[-_.]\d+)?$// ) { - return $index; + if( my $base = $index =~ s/[^a-z0-9]+$PATTERN_REGEX{DATE}.*$//rio ) { + return $base; } return; } @@ -1106,6 +1096,8 @@ sub es_index_bases { # Strip to the base my $stripped = es_index_strip_date($index); + # Remove the rollover portion + $stripped =~ s/[\-_.]\d+$//; return unless defined $stripped and length $stripped; # Compute if we haven't already memoized @@ -1132,7 +1124,6 @@ Return the number of days old this index is. =cut -my $NOW = timegm(0,0,0,(gmtime)[3,4,5]); sub es_index_days_old { my ($index) = @_; @@ -1140,24 +1131,32 @@ sub es_index_days_old { es_utils_initialize() unless keys %DEF; - if( my ($dateStr) = ($index =~ /($PATTERN_REGEX{DATE})/) ) { - my @date=(); - if(length $DEF{DATESEP}) { - @date = reverse map { int } split /\Q$DEF{DATESEP}\E/, $dateStr; - } - else { - for my $len (qw(4 2 2)) { - unshift @date, substr($dateStr,0,$len,''); - } - } + + if( $index =~ /[^a-z0-9]$PATTERN_REGEX{DATE}/io ) { + # Build Date Array + my @date = map { int } + grep { length } + map { $+{$_} =~ s/^0//r } qw(day month year); $date[1]--; # move 1-12 -> 0-11 + # Validate + if( @date != 3 ) { + warn sprintf "es_index_days_old(%s) matched DATE(%s), but did not receive enough parts: %s", + $index, + $+{datestr}, + join(', ', map { "'$_'" } @date); + return; + } + + # Calculate Difference + my $now = timegm(0,0,0,(gmtime)[3,4,5]); my $idx_time = eval { timegm( 0,0,0, @date ) }; return unless $idx_time; - my $diff = $NOW - $idx_time; + my $diff = $now - $idx_time; $diff++; # Add one second debug({color=>"yellow"}, sprintf "es_index_days_old(%s) - Time difference is %0.3f", $index, $diff/86400); return int($diff / 86400); } + verbose({color=>"red"}, "es_index_days_old($index) - date string not found"); return; } @@ -1570,9 +1569,9 @@ sub es_local_index_meta { es_utils_initialize() unless keys %DEF; if( exists $_GLOBALS{meta} ) { - my $meta = $_GLOBALS{meta}; + my $meta = $_GLOBALS{meta}; my @search = ( $name_or_base ); - push @search, es_index_strip_date( $name_or_base ); + push @search, es_index_strip_date($name_or_base); push @search, es_index_bases($name_or_base); foreach my $check ( @search ) { diff --git a/scripts/es-daily-index-maintenance.pl b/scripts/es-daily-index-maintenance.pl index 856c109..8b0e316 100755 --- a/scripts/es-daily-index-maintenance.pl +++ b/scripts/es-daily-index-maintenance.pl @@ -28,7 +28,6 @@ 'optimize', 'optimize-days=i', 'index-basename=s', - 'date-separator=s', 'timezone=s', 'skip-alias=s@', # Basic options diff --git a/scripts/es-graphite-dynamic.pl b/scripts/es-graphite-dynamic.pl index c13583f..5d983be 100755 --- a/scripts/es-graphite-dynamic.pl +++ b/scripts/es-graphite-dynamic.pl @@ -73,7 +73,6 @@ #------------------------------------------------------------------------# # Collect and Decode the Cluster Statistics - my @metrics = sort map { "$_->{key} $_->{value}" } @{ $Fetcher->get_metrics }; if( !@metrics ) { output({color=>'red'}, "Error retrieving metrics"); diff --git a/t/02-index-data.t b/t/02-index-data.t index 284d094..f24a7e8 100644 --- a/t/02-index-data.t +++ b/t/02-index-data.t @@ -16,7 +16,18 @@ $Data::Dumper::Sortkeys = 1; my $now = DateTime->now(); my @days_old = qw(0 1 3 5 8 13 21 90); -my %TESTS=(); +my %TESTS=( + 'notadate-100000' => { + es_index_bases => 'notadate', + es_index_days_old => undef, + es_index_strip_date => 'notadate-100000', + }, + strftime('mystery-science-theater-3000-%Y.%m.%d', localtime) => { + es_index_bases => 'mystery,mystery-science,mystery-science-theater', + es_index_days_old => 0, + es_index_strip_date => 'mystery-science-theater-3000', + }, +); foreach my $days_old ( @days_old ) { # Query String Parser Testing my $lt = $now->clone->subtract( days => $days_old ); @@ -37,6 +48,11 @@ foreach my $days_old ( @days_old ) { es_index_days_old => $days_old, es_index_strip_date => 'type_dcid', }, + "type_dcid_$date-0001" => { + es_index_bases => 'type,type_dcid', + es_index_days_old => $days_old, + es_index_strip_date => 'type_dcid', + }, ); # Install the test globally foreach my $t (keys %tests) { @@ -50,7 +66,7 @@ foreach my $t (sort keys %TESTS) { my $got = { es_index_bases => join(',', es_index_bases($t)), es_index_strip_date => es_index_strip_date($t), - es_index_days_old => es_index_days_old($t), + es_index_days_old => es_index_days_old($t) // undef, }; is_deeply($got,$TESTS{$t},sprintf "%s - %s", $t, join(',', sort keys %{$got})) or diag( Dumper $got ); }