From b730c4e49ae72a26ed8eed2a19bdfcedf3380cbb Mon Sep 17 00:00:00 2001 From: xuejiaming <326308290@qq.com> Date: Sat, 20 Apr 2024 08:39:15 +0800 Subject: [PATCH] =?UTF-8?q?[#266]=20=E6=B7=BB=E5=8A=A0GetCompareValueBySha?= =?UTF-8?q?rdingKey=E6=96=B9=E6=B3=95=E4=BC=98=E5=8C=96=E5=BD=93=E5=87=BA?= =?UTF-8?q?=E7=8E=B0=E5=A4=A7=E9=87=8FIn=E6=97=B6Expression=E7=9A=84or?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E6=88=96=E8=80=85and=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E6=8B=BC=E6=8E=A5=E5=AF=BC=E8=87=B4stackoverflow=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- samples/Sample.AutoCreateIfPresent/Program.cs | 4 ++++ .../Controllers/WeatherForecastController.cs | 13 ++++++++++++- .../Shardings/SysUserModVirtualTableRoute.cs | 9 +++++++++ samples/Sample.MySql/Startup.cs | 2 +- ...bstractShardingOperatorVirtualDataSourceRoute.cs | 7 ++++++- .../AbstractShardingOperatorVirtualTableRoute.cs | 6 +++++- .../Visitors/QueryableRouteDiscoverVisitor.cs | 11 ++++++++++- src/ShardingCore/Utils/ShardingUtil.cs | 4 ++-- .../ShardingDataSourceMod.cs | 6 +++++- test/ShardingCore.CommonTest/ShardingTableTime.cs | 7 ++++++- 10 files changed, 60 insertions(+), 9 deletions(-) diff --git a/samples/Sample.AutoCreateIfPresent/Program.cs b/samples/Sample.AutoCreateIfPresent/Program.cs index 3a472cde..089aaa32 100644 --- a/samples/Sample.AutoCreateIfPresent/Program.cs +++ b/samples/Sample.AutoCreateIfPresent/Program.cs @@ -30,6 +30,10 @@ }) .UseConfig(o => { + // o.CacheEntrySize; + // o.CacheModelLockConcurrencyLevel + // o.CacheModelLockObjectSeconds + // o.CacheItemPriority o.ThrowIfQueryRouteNotMatch = false; o.AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=shardingTest;userid=root;password=root;"); o.UseShardingQuery((conn, b) => diff --git a/samples/Sample.MySql/Controllers/WeatherForecastController.cs b/samples/Sample.MySql/Controllers/WeatherForecastController.cs index 3d45d89c..9977f98d 100644 --- a/samples/Sample.MySql/Controllers/WeatherForecastController.cs +++ b/samples/Sample.MySql/Controllers/WeatherForecastController.cs @@ -131,6 +131,7 @@ public IQueryable GetAll() [HttpGet] public async Task Getxx() { + var test = new Test(); test.UtcTime=DateTime.Now; await _defaultTableDbContext.AddAsync(test); @@ -141,7 +142,7 @@ public async Task Getxx() public async Task Get() { var s = Guid.NewGuid().ToString(); - var page =await _defaultTableDbContext.Set().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2); + // var page =await _defaultTableDbContext.Set().Include().ThenInclude().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2); // var virtualDataSource = _shardingRuntimeContext.GetVirtualDataSource(); // virtualDataSource.AddPhysicDataSource(new DefaultPhysicDataSource("2023", "xxxxxxxx", false)); // var dataSourceRouteManager = _shardingRuntimeContext.GetDataSourceRouteManager(); @@ -571,5 +572,15 @@ public void get11() unShardingDbContext2.SaveChanges(); dbContextTransaction.Commit(); } + + [HttpGet] + public async Task get131() + { + var list = new List(); + var idList = Enumerable.Range(1,50000).Select(o=>o.ToString()).ToList(); + var sysUserMods = _defaultTableDbContext.Set() + .Where(o=>idList.Contains(o.Id)).ToList(); + return Ok(); + } } } diff --git a/samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs b/samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs index 561971b9..b4088070 100644 --- a/samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs +++ b/samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs @@ -24,6 +24,15 @@ public override void Configure(EntityMetadataTableBuilder builder) builder.ShardingProperty(o => o.Id); } + public override object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName) + { + if ("Id".Equals(shardingPropertyName)) + { + return ShardingKeyToTail(shardingKey); + } + return base.GetCompareValueByShardingKey(shardingKey, shardingPropertyName); + } + // protected override List AfterShardingRouteUnitFilter(DataSourceRouteResult dataSourceRouteResult, List shardingRouteUnits) // { // //拦截 diff --git a/samples/Sample.MySql/Startup.cs b/samples/Sample.MySql/Startup.cs index 684dd8ca..7091de91 100644 --- a/samples/Sample.MySql/Startup.cs +++ b/samples/Sample.MySql/Startup.cs @@ -226,7 +226,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.UseDeveloperExceptionPage(); } - app.ApplicationServices.UseAutoTryCompensateTable(); using (var scope = app.ApplicationServices.CreateScope()) { var unShardingDbContext = scope.ServiceProvider.GetService(); @@ -236,6 +235,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) // var virtualTableRoute = (SysUserLogByMonthRoute)tableRouteManager.GetRoute(typeof(SysUserLogByMonth)); // virtualTableRoute.Append("2021"); } + app.ApplicationServices.UseAutoTryCompensateTable(); // var shardingRuntimeContext = app.ApplicationServices.GetRequiredService(); // var entityMetadataManager = shardingRuntimeContext.GetEntityMetadataManager(); // var entityMetadata = entityMetadataManager.TryGet(); diff --git a/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/Abstractions/AbstractShardingOperatorVirtualDataSourceRoute.cs b/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/Abstractions/AbstractShardingOperatorVirtualDataSourceRoute.cs index d19bf616..11fa6a81 100644 --- a/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/Abstractions/AbstractShardingOperatorVirtualDataSourceRoute.cs +++ b/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/Abstractions/AbstractShardingOperatorVirtualDataSourceRoute.cs @@ -24,7 +24,7 @@ public abstract class AbstractShardingOperatorVirtualDataSourceRoute DoRouteWithPredicate(List allDataSourceNames, IQueryable queryable) { //获取路由后缀表达式 - var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter, false); + var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false); //表达式缓存编译 // var filter = CachingCompile(routeParseExpression); var filter = routeParseExpression.GetRoutePredicate(); @@ -33,6 +33,11 @@ protected override List DoRouteWithPredicate(List allDataSourceN return dataSources; } + public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName) + { + return shardingKey; + } + /// /// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表 diff --git a/src/ShardingCore/Core/VirtualRoutes/TableRoutes/Abstractions/AbstractShardingOperatorVirtualTableRoute.cs b/src/ShardingCore/Core/VirtualRoutes/TableRoutes/Abstractions/AbstractShardingOperatorVirtualTableRoute.cs index 677dcdf5..2a7acd93 100644 --- a/src/ShardingCore/Core/VirtualRoutes/TableRoutes/Abstractions/AbstractShardingOperatorVirtualTableRoute.cs +++ b/src/ShardingCore/Core/VirtualRoutes/TableRoutes/Abstractions/AbstractShardingOperatorVirtualTableRoute.cs @@ -27,7 +27,7 @@ public abstract class AbstractShardingOperatorVirtualTableRoute : protected override List DoRouteWithPredicate(DataSourceRouteResult dataSourceRouteResult, IQueryable queryable) { //获取路由后缀表达式 - var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,true); + var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey,true); //表达式缓存编译 // var filter =CachingCompile(routeParseExpression); var filter =routeParseExpression.GetRoutePredicate(); @@ -40,6 +40,10 @@ protected override List DoRouteWithPredicate(DataSourceRouteResu return sqlRouteUnits; } + public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName) + { + return shardingKey; + } /// /// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表 diff --git a/src/ShardingCore/Sharding/Visitors/QueryableRouteDiscoverVisitor.cs b/src/ShardingCore/Sharding/Visitors/QueryableRouteDiscoverVisitor.cs index 5aa1c128..37927719 100644 --- a/src/ShardingCore/Sharding/Visitors/QueryableRouteDiscoverVisitor.cs +++ b/src/ShardingCore/Sharding/Visitors/QueryableRouteDiscoverVisitor.cs @@ -51,6 +51,7 @@ public class QueryableRouteShardingTableDiscoverVisitor : ShardingExpressionVisi private readonly EntityMetadata _entityMetadata; private readonly Func> _keyToTailWithFilter; + private readonly Func _compareValueByKey; /// /// 是否是分表路由 @@ -63,10 +64,11 @@ public class QueryableRouteShardingTableDiscoverVisitor : ShardingExpressionVisi private RoutePredicateExpression _where = RoutePredicateExpression.Default; public QueryableRouteShardingTableDiscoverVisitor(EntityMetadata entityMetadata, - Func> keyToTailWithFilter, bool shardingTableRoute) + Func> keyToTailWithFilter,Func compareValueByKey, bool shardingTableRoute) { _entityMetadata = entityMetadata; _keyToTailWithFilter = keyToTailWithFilter; + _compareValueByKey = compareValueByKey; _shardingTableRoute = shardingTableRoute; } @@ -378,8 +380,15 @@ private RoutePredicateExpression ResolveInFunc(MethodCallExpression methodCallEx if (arrayObject is IEnumerable enumerableObj) { + var compareSet = new HashSet(); foreach (var shardingValue in enumerableObj) { + var compareValueByKey = _compareValueByKey(shardingValue,shardingPredicateResult.ShardingPropertyName); + if (!compareSet.Add(compareValueByKey)) + { + continue; + } + var eq = _keyToTailWithFilter(shardingValue, @in ? ShardingOperatorEnum.Equal : ShardingOperatorEnum.NotEqual, shardingPredicateResult.ShardingPropertyName); diff --git a/src/ShardingCore/Utils/ShardingUtil.cs b/src/ShardingCore/Utils/ShardingUtil.cs index 90cafae8..8910630a 100644 --- a/src/ShardingCore/Utils/ShardingUtil.cs +++ b/src/ShardingCore/Utils/ShardingUtil.cs @@ -34,10 +34,10 @@ public class ShardingUtil /// /// sharding table or data source /// - public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func> keyToTailExpression,bool shardingTableRoute) + public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func> keyToTailExpression,Func compareValueByKey,bool shardingTableRoute) { - QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression, shardingTableRoute); + QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression,compareValueByKey, shardingTableRoute); visitor.Visit(queryable.Expression); diff --git a/test/ShardingCore.CommonTest/ShardingDataSourceMod.cs b/test/ShardingCore.CommonTest/ShardingDataSourceMod.cs index 81b59e92..bca59159 100644 --- a/test/ShardingCore.CommonTest/ShardingDataSourceMod.cs +++ b/test/ShardingCore.CommonTest/ShardingDataSourceMod.cs @@ -24,6 +24,10 @@ public ShardingDataSourceMod() _allDataSources = Enumerable.Range(0, 10).Select(o => o.ToString()).ToList(); } + public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName) + { + return shardingKey; + } public static Func GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator, string propertyName) { @@ -47,7 +51,7 @@ public static Func GetRouteFilter(object shardingValue, ShardingOp private void TestId(IQueryable queryable, string[] dataSourceNames) { var routePredicateExpression = - ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter, false); + ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false); Assert.NotNull(routePredicateExpression); var routePredicate = routePredicateExpression.GetRoutePredicate(); diff --git a/test/ShardingCore.CommonTest/ShardingTableTime.cs b/test/ShardingCore.CommonTest/ShardingTableTime.cs index f4f7d19c..72c2b7c2 100644 --- a/test/ShardingCore.CommonTest/ShardingTableTime.cs +++ b/test/ShardingCore.CommonTest/ShardingTableTime.cs @@ -25,6 +25,11 @@ public ShardingTableTime() //[20220101....20220120] _allTables = Enumerable.Range(0,20).Select(o=>dateTime.AddDays(o).ToString("yyyyMMdd")).ToList(); } + + public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName) + { + return shardingKey; + } public static Func GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator, string propertyName) { @@ -60,7 +65,7 @@ public static Func GetRouteFilter(object shardingValue, ShardingOp private void TestId(IQueryable queryable, string[] tables) { - var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,true); + var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,GetCompareValueByShardingKey,true); Assert.NotNull(routePredicateExpression); var routePredicate = routePredicateExpression.GetRoutePredicate();