feat: add support for filter config (#860)

README.md

@@ -8,6 +8,7 @@
 - [English Documentation](https://tair-opensource.github.io/RedisShake/en/)
 
 ![](./docs/demo.gif)
+
 ## Overview
 
 RedisShake is a tool designed for processing and migrating Redis data. It offers the following features:
@@ -50,22 +51,27 @@ sh build.sh
 
 ### Usage
 
-Assume you have two Redis instances:
+To migrate data from one Redis instance to another while skipping keys with specific prefixes, follow these steps:
+
+1. Ensure you have two Redis instances running:
 
 * Instance A: 127.0.0.1:6379
 * Instance B: 127.0.0.1:6380
 
-Create a new configuration file `shake.toml`:
+2. Create a new configuration file `shake.toml`, and set the `block_key_prefix` parameter to skip keys with specific prefixes:
 
 ```toml
 [sync_reader]
 address = "127.0.0.1:6379"
 
 [redis_writer]
 address = "127.0.0.1:6380"
+
+[filter]
+block_key_prefix = ["temp:", "cache:"]
 ```
 
-To start RedisShake, run the following command:
+3. Start RedisShake by running the following command:
 
 ```shell
 ./redis-shake shake.toml
@@ -84,6 +90,7 @@ would like to change. We are particularly interested in:
 1. Adding support for more modules
 2. Enhancing support for Readers and Writers
 3. Sharing your Lua scripts and best practices
+4. Improving the documentation
 
 ## History
 

cmd/redis-shake/main.go

@@ -10,7 +10,7 @@ import (
 
 	"RedisShake/internal/config"
 	"RedisShake/internal/entry"
-	"RedisShake/internal/function"
+	"RedisShake/internal/filter"
 	"RedisShake/internal/log"
 	"RedisShake/internal/reader"
 	"RedisShake/internal/status"
@@ -27,7 +27,7 @@ func main() {
 	utils.ChdirAndAcquireFileLock()
 	utils.SetNcpu()
 	utils.SetPprofPort()
-	luaRuntime := function.New(config.Opt.Function)
+	luaRuntime := filter.NewFunctionFilter(config.Opt.Filter.Function)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -139,6 +139,10 @@ Loop:
 			status.AddReadCount(e.CmdName)
 
 			// filter
+			if !filter.Filter(e) {
+				log.Debugf("skip command: %v", e)
+				continue
+			}
 			log.Debugf("function before: %v", e)
 			entries := luaRuntime.RunFunction(e)
 			log.Debugf("function after: %v", entries)

docs/.vitepress/en.ts

@@ -48,16 +48,18 @@ function sidebar(): DefaultTheme.SidebarItem[] {
             ]
         },
         {
-            text: 'Function',
+            text: 'Filter and Processing',
             items: [
-                { text: 'What is function', link: '/en/function/introduction' },
-                { text: 'Best Practices', link: '/en/function/best_practices' }
+                { text: 'Built-in Filter Rules', link: '/en/filter/filter' },
+                { text: 'What is function', link: '/en/filter/function' },
             ]
         },
         {
             text: 'Others',
             items: [
                 { text: 'Redis Modules', link: '/en/others/modules' },
+                { text: 'How to Verify Data Consistency', link: '/en/others/consistent' },
+                { text: 'Cross-version Migration', link: '/en/others/version' },
             ]
         },
     ]

docs/.vitepress/zh.ts

@@ -46,10 +46,10 @@ function sidebar(): DefaultTheme.SidebarItem[] {
             ]
         },
         {
-            text: 'Function',
+            text: '过滤与加工',
             items: [
-                { text: '什么是 function', link: '/zh/function/introduction' },
-                { text: '最佳实践', link: '/zh/function/best_practices' }
+                { text: '内置过滤规则', link: '/zh/filter/filter' },
+                { text: '什么是 function', link: '/zh/filter/function' },
             ]
         },
         {

docs/src/en/filter/filter.md

@@ -0,0 +1,40 @@
+---
+outline: deep
+---
+# Built-in Filter Rules
+RedisShake provides various built-in filter rules that users can choose from according to their needs.
+
+## Filtering Keys
+RedisShake supports filtering by key name prefixes and suffixes. You can set the following options in the configuration file, for example:
+```toml
+allow_key_prefix = ["user:", "product:"]
+allow_key_suffix = [":active", ":valid"]
+block_key_prefix = ["temp:", "cache:"]
+block_key_suffix = [":tmp", ":old"]
+```
+If these options are not set, all keys are allowed by default.
+
+## Filtering Databases
+You can specify allowed or blocked database numbers, for example:
+```toml
+allow_db = [0, 1, 2]
+block_db = [3, 4, 5]
+```
+If these options are not set, all databases are allowed by default.
+
+## Filtering Commands
+RedisShake allows you to filter specific Redis commands, for example:
+```toml
+allow_command = ["GET", "SET"]
+block_command = ["DEL", "FLUSHDB"]
+``` 
+
+## Filtering Command Groups
+
+You can also filter by command groups. Available command groups include:
+SERVER, STRING, CLUSTER, CONNECTION, BITMAP, LIST, SORTED_SET, GENERIC, TRANSACTIONS, SCRIPTING, TAIRHASH, TAIRSTRING, TAIRZSET, GEO, HASH, HYPERLOGLOG, PUBSUB, SET, SENTINEL, STREAM
+For example:
+```toml
+allow_command_group = ["STRING", "HASH"]
+block_command_group = ["SCRIPTING", "PUBSUB"]
+```

docs/src/en/guide/config.md

@@ -4,79 +4,43 @@ outline: deep
 
 # Configuration File
 
-RedisShake uses the [TOML](https://toml.io/cn/) language for writing, and all configuration parameters are explained in all.toml.
+RedisShake uses the [TOML](https://toml.io/) language for writing, and all configuration parameters are explained in all.toml.
 
 The configuration file is composed as follows:
 
 ```toml
-function = "..."
-
 [xxx_reader]
 ...
 
 [xxx_writer]
 ...
 
+[filter]
+...
+
 [advanced]
 ...
 ```
 
-Under normal usage, you only need to write the `xxx_reader` and `xxx_writer` parts. The `function` and `advanced` parts are for advanced usage, and users can configure them according to their needs.
-
-## function Configuration
-
-Refer to [What is function](../function/introduction.md).
-
 ## reader Configuration
 
 RedisShake provides different Readers to interface with different sources, see the Reader section for configuration details:
 
 * [Sync Reader](../reader/sync_reader.md)
 * [Scan Reader](../reader/scan_reader.md)
 * [RDB Reader](../reader/rdb_reader.md)
+* [AOF Reader](../reader/aof_reader.md)
 
 ## writer Configuration
 
 RedisShake provides different Writers to interface with different targets, see the Writer section for configuration details:
 
 * [Redis Writer](../writer/redis_writer.md)
 
+## filter Configuration
+
+You can set filter rules through the configuration file. Refer to [Filter and Processing](../filter/filter.md) and [function](../filter/function.md).
+
 ## advanced Configuration
 
-```toml
-[advanced]
-dir = "data"
-ncpu = 3 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
-
-pprof_port = 0 # pprof port, 0 means disable
-status_port = 0 # status port, 0 means disable
-
-# log
-log_file = "shake.log"
-log_level = "info" # debug, info or warn
-log_interval = 5 # in seconds
-
-# redis-shake gets key and value from rdb file, and uses RESTORE command to
-# create the key in target redis. Redis RESTORE will return a "Target key name
-# is busy" error when key already exists. You can use this configuration item
-# to change the default behavior of restore:
-# panic:   redis-shake will stop when meet "Target key name is busy" error.
-# rewrite: redis-shake will replace the key with new value.
-# ignore:  redis-shake will skip restore the key when meet "Target key name is busy" error.
-rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip
-
-# redis-shake uses pipeline to improve sending performance.
-# This item limits the maximum number of commands in a pipeline.
-pipeline_count_limit = 1024
-
-# Client query buffers accumulate new commands. They are limited to a fixed
-# amount by default. This amount is normally 1gb.
-target_redis_client_max_querybuf_len = 1024_000_000
-
-# In the Redis protocol, bulk requests, that are, elements representing single
-# strings, are normally limited to 512 mb.
-target_redis_proto_max_bulk_len = 512_000_000
-
-# If the source is Elasticache or MemoryDB, you can set this item.
-aws_psync = ""
-```
+Refer to the [shake.toml configuration file](https://github.com/tair-opensource/RedisShake/blob/v4/shake.toml).