Skip to content

Commit

Permalink
优化 JOIN 副表解析结果集 ResultSet 的性能（减少同副表字段的重复逻辑）
Browse files Browse the repository at this point in the history
  • Loading branch information
@TommyLemon
TommyLemon committed Jan 2, 2022
1 parent 61883bc commit 5d59b35
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 85 deletions.
15 changes: 15 additions & 0 deletions APIJSONORM/src/main/java/apijson/StringUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.File;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.util.Objects;
import java.util.regex.Pattern;

/**通用字符串(String)相关类,为null时返回""
Expand Down Expand Up @@ -891,4 +892,18 @@ public static String concat(String left, String right, String split, boolean tri

//校正（自动补全等）字符串>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>


public static boolean equals(Object s1, Object s2) {
return Objects.equals(s1, s2);
}
public static boolean equalsIgnoreCase(String s1, String s2) {
if (s1 == s2) {
return true;
}
if (s1 == null || s2 == null) {
return false;
}
return s1.equalsIgnoreCase(s2);
}

}
5 changes: 3 additions & 2 deletions APIJSONORM/src/main/java/apijson/orm/AbstractParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1851,7 +1851,8 @@ public JSONObject executeSQL(SQLConfig config, boolean isSubquery) throws Except
return result;
}
catch (Exception e) {
if (Log.DEBUG && e.getMessage().contains(Log.KEY_SYSTEM_INFO_DIVIDER) == false) {
String msg = e.getMessage();
if (Log.DEBUG && msg != null && msg.contains(Log.KEY_SYSTEM_INFO_DIVIDER) == false) {
try {
String db = config.getDatabase();
if (db == null) {
Expand Down Expand Up @@ -1880,7 +1881,7 @@ else if (config.isClickHouse()) {

Class<? extends Exception> clazz = e.getClass();
e = clazz.getConstructor(String.class).newInstance(
e.getMessage()
msg
+ " " + Log.KEY_SYSTEM_INFO_DIVIDER + " \n **环境信息** "
+ " \n 系统: " + System.getProperty("os.name") + " " + System.getProperty("os.version")
+ " \n 数据库: " + db + " " + config.getDBVersion()
Expand Down
148 changes: 68 additions & 80 deletions APIJSONORM/src/main/java/apijson/orm/AbstractSQLExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public int getExecutedSQLCount() {
* @param isStatic
*/
@Override
public synchronized void putCache(String sql, List<JSONObject> list, int type) {
public void putCache(String sql, List<JSONObject> list, int type) {
if (sql == null || list == null) { //空map有效，说明查询过sql了 || list.isEmpty()) {
Log.i(TAG, "saveList sql == null || list == null >> return;");
return;
Expand All @@ -90,7 +90,7 @@ public synchronized void putCache(String sql, List<JSONObject> list, int type) {
* @param isStatic
*/
@Override
public synchronized void removeCache(String sql, int type) {
public void removeCache(String sql, int type) {
if (sql == null) {
Log.i(TAG, "removeList sql == null >> return;");
return;
Expand All @@ -115,7 +115,8 @@ public JSONObject getCacheItem(String sql, int position, int type) {
//只要map不为null，则如果 list.get(position) == null，则返回 {} ，避免再次SQL查询
if (list == null) {
return null;
}
}

JSONObject result = position >= list.size() ? null : list.get(position);
return result != null ? result : new JSONObject();
}
Expand All @@ -124,20 +125,14 @@ public JSONObject getCacheItem(String sql, int position, int type) {

@Override
public ResultSet executeQuery(@NotNull Statement statement, String sql) throws Exception {
executedSQLCount ++;

return statement.executeQuery(sql);
}
@Override
public int executeUpdate(@NotNull Statement statement, String sql) throws Exception {
executedSQLCount ++;

return statement.executeUpdate(sql);
}
@Override
public ResultSet execute(@NotNull Statement statement, String sql) throws Exception {
executedSQLCount ++;

statement.execute(sql);
return statement.getResultSet();
}
Expand Down Expand Up @@ -186,6 +181,10 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
try {
if (unknowType) {
Statement statement = getStatement(config);

if (isExplain == false) { //只有 SELECT 才能 EXPLAIN
executedSQLCount ++;
}
rs = execute(statement, sql);

result = new JSONObject(true);
Expand All @@ -199,8 +198,9 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
case POST:
case PUT:
case DELETE:
executedSQLCount ++;

if (isExplain == false) { //只有 SELECT 才能 EXPLAIN
executedSQLCount ++;
}
int updateCount = executeUpdate(config);
if (updateCount <= 0) {
throw new IllegalAccessException("没权限访问或对象不存在！"); // NotExistException 会被 catch 转为成功状态
Expand All @@ -222,7 +222,7 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
case GETS:
case HEAD:
case HEADS:
result = isHead ? null : getCacheItem(sql, position, config.getCache());
result = isHead || isExplain ? null : getCacheItem(sql, position, config.getCache());
Log.i(TAG, ">>> execute result = getCache('" + sql + "', " + position + ") = " + result);
if (result != null) {
cachedSQLCount ++;
Expand All @@ -231,11 +231,10 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
return result;
}

rs = executeQuery(config); //FIXME SQL Server 是一次返回两个结果集，包括查询结果和执行计划，需要 moreResults

if (isExplain == false) { //只有 SELECT 才能 EXPLAIN
executedSQLCount ++;
}
rs = executeQuery(config); //FIXME SQL Server 是一次返回两个结果集，包括查询结果和执行计划，需要 moreResults
break;

default://OPTIONS, TRACE等
Expand Down Expand Up @@ -270,6 +269,7 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
childMap = new HashMap<>(); //要存到cacheMap
// Map<Integer, Join> columnIndexAndJoinMap = new HashMap<>(length);
String lastTableName = null; // 默认就是主表 config.getTable();
String lastAliasName = null; // 默认就是主表 config.getAlias();
int lastViceTableStart = 0;
int lastViceColumnStart = 0;
Join lastJoin = null;
Expand All @@ -289,6 +289,7 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
Log.d(TAG, "\n\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n execute while (rs.next()){ index = " + index + "\n\n");

JSONObject item = new JSONObject(true);
JSONObject curItem = item;
boolean isMain = true;

for (int i = 1; i <= length; i++) {
Expand All @@ -301,17 +302,19 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
Join curJoin = columnIndexAndJoinMap == null ? null : columnIndexAndJoinMap[i - 1]; // columnIndexAndJoinMap.get(i);

// 为什么 isExplain == false 不用判断？因为所有字段都在一张 Query Plan 表
if (index <= 0 && hasJoin && ! isExplain) { // && viceColumnStart > length) {
if (index <= 0 && columnIndexAndJoinMap != null) { // && viceColumnStart > length) {

SQLConfig curConfig = curJoin == null || ! curJoin.isSQLJoin() ? null : curJoin.getCacheConfig();
List<String> curColumn = curConfig == null ? null : curConfig.getColumn();
String sqlTable = curConfig == null ? null : curConfig.getSQLTable();
String sqlAlias = curConfig == null ? null : curConfig.getAlias();

List<String> column = config.getColumn();
int mainColumnSize = column == null ? 0 : column.size();
// FIXME 主副表同名导致主表数据当成副表数据 { "[]": { "join": "</Comment:to/id@", "Comment": { "toId>": 0 }, "Comment:to": { "@column": "id,content", "id@": "/Comment/toId" } }, "@explain": true }
boolean toFindJoin = mainColumnSize <= 0 || i > mainColumnSize; // 主表就不用找 JOIN 配置

if (StringUtil.isEmpty(sqlTable , true)) {
if (StringUtil.isEmpty(sqlTable, true)) {
if (toFindJoin) { // 在主表字段数量内的都归属主表
sqlTable = rsmd.getTableName(i); // SQL 函数甚至部分字段都不返回表名，当然如果没传 @column 生成的 Table.* 则返回的所有字段都会带表名

Expand All @@ -323,12 +326,15 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
SQLConfig cfg = join == null || ! join.isSQLJoin() ? null : join.getJoinConfig();
List<String> c = cfg == null ? null : cfg.getColumn();

nextViceColumnStart += (
c != null && ! c.isEmpty() ? c.size()
: (Objects.equals(sqlTable, lastTableName) || sqlTable.equalsIgnoreCase(lastTableName) ? 1 : 0)
nextViceColumnStart += (c != null && ! c.isEmpty() ?
c.size() : (
StringUtil.equalsIgnoreCase(sqlTable, lastTableName)
&& StringUtil.equals(sqlAlias, lastAliasName) ? 1 : 0
)
);
if (i < nextViceColumnStart) {
sqlTable = cfg.getSQLTable();
sqlAlias = cfg.getAlias();
lastViceTableStart = j; // 避免后面的空 @column 表内字段被放到之前的空 @column 表

curJoin = join;
Expand All @@ -345,6 +351,7 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknowType) throws
// 没有 @column，仍然定位不了，用前一个 table 名。FIXME 如果刚好某个表内第一个字段是就是 SQL 函数？
if (StringUtil.isEmpty(sqlTable, true)) {
sqlTable = lastTableName;
sqlAlias = lastAliasName;
toFindJoin = false;
}
}
Expand All @@ -353,16 +360,18 @@ else if (config.isClickHouse() && (sqlTable.startsWith("`") || sqlTable.startsWi
sqlTable = sqlTable.substring(1, sqlTable.length() - 1);
}

if ((sqlTable == null && lastTableName != null) || (sqlTable != null && ! sqlTable.equalsIgnoreCase(lastTableName))) {
if (StringUtil.equalsIgnoreCase(sqlTable, lastTableName) == false || StringUtil.equals(sqlAlias, lastAliasName) == false) {
lastTableName = sqlTable;
lastAliasName = sqlAlias;
lastViceColumnStart = i;

if (toFindJoin) { // 找到对应的副表 JOIN 配置
for (int j = lastViceTableStart; j < joinList.size(); j++) { // 查找副表 @column，定位字段所在表
Join join = joinList.get(j);
SQLConfig cfg = join == null || ! join.isSQLJoin() ? null : join.getJoinConfig();

if (cfg != null && sqlTable != null && sqlTable.equalsIgnoreCase(cfg.getSQLTable())) {
if (cfg != null && StringUtil.equalsIgnoreCase(sqlTable, cfg.getSQLTable())
) { // FIXME 导致副表字段错放到主表 && StringUtil.equals(sqlAlias, cfg.getAlias())) {
lastViceTableStart = j; // 避免后面的空 @column 表内字段被放到之前的空 @column 表

curJoin = join;
Expand All @@ -378,7 +387,7 @@ else if (config.isClickHouse() && (sqlTable.startsWith("`") || sqlTable.startsWi

if (isMain) {
lastViceColumnStart ++;
}
}
else {
if (curJoin == null) {
curJoin = lastJoin;
Expand All @@ -398,23 +407,42 @@ else if (config.isClickHouse() && (sqlTable.startsWith("`") || sqlTable.startsWi
}
}

columnIndexAndJoinMap[i - 1] = curJoin; // columnIndexAndJoinMap.put(i, curJoin); // TODO columnIndexAndTableMap[i] = sqlTable 提速?

// if (column != null && column.isEmpty() == false) {
// viceColumnStart = column.size() + 1;
// }
// else if (config.getSQLTable().equalsIgnoreCase(sqlTable) == false) {
// viceColumnStart = i;
// }
columnIndexAndJoinMap[i - 1] = curJoin;
}

// 如果是主表则直接用主表对应的 item，否则缓存副表数据到 childMap
Join prevJoin = columnIndexAndJoinMap == null || i < 2 ? null : columnIndexAndJoinMap[i - 2];
if (curJoin != prevJoin) { // 前后字段不在同一个表对象，即便后面出现 null，也不该是主表数据，而是逻辑 bug 导致
SQLConfig viceConfig = curJoin != null && curJoin.isSQLJoin() ? curJoin.getCacheConfig() : null;
if (viceConfig != null) { //FIXME 只有和主表关联才能用 item，否则应该从 childMap 查其它副表数据
viceConfig.putWhere(curJoin.getKey(), item.get(curJoin.getTargetKey()), true);
}
String viceSql = viceConfig == null ? null : viceConfig.getSQL(false); //TODO 在 SQLConfig 缓存 SQL，减少大量的重复生成

item = onPutColumn(config, rs, rsmd, index, item, i, curJoin, childMap); // isExplain == false && hasJoin && i >= viceColumnStart ? childMap : null);
if (StringUtil.isEmpty(viceSql, true)) {
Log.i(TAG, "execute StringUtil.isEmpty(viceSql, true) >> item = null; >> ");
curItem = null;
}
else if (curJoin.isOuterJoin() || curJoin.isAntiJoin()) {
Log.i(TAG, "execute curJoin.isOuterJoin() || curJoin.isAntiJoin() >> item = null; >> ");
curItem = null; // 肯定没有数据，缓存也无意义
// 副表是按常规条件查询，缓存会导致其它同表同条件对象查询结果集为空 childMap.put(viceSql, new JSONObject()); // 缓存固定空数据，避免后续多余查询
}
else {
curItem = (JSONObject) childMap.get(viceSql);
if (curItem == null) {
curItem = new JSONObject(true);
childMap.put(viceSql, curItem);
}
}
}

curItem = onPutColumn(config, rs, rsmd, index, curItem, i, curJoin, childMap); // isExplain == false && hasJoin && i >= viceColumnStart ? childMap : null);
}

resultList = onPutTable(config, rs, rsmd, resultList, index, item);

Log.d(TAG, "\n execute while (rs.next()) { resultList.put( " + index + ", result); "
+ "\n >>>>>>>>>>>>>>>>>>>>>>>>>>> \n\n");
Log.d(TAG, "execute while (rs.next()) { resultList.put( " + index + ", result); " + "\n >>>>>>>>>>>>>>>>>>>>>>>>>>> \n\n");
}
}
}
Expand Down Expand Up @@ -622,61 +650,21 @@ protected void executeAppJoin(SQLConfig config, List<JSONObject> resultList, Map
*/
protected JSONObject onPutColumn(@NotNull SQLConfig config, @NotNull ResultSet rs, @NotNull ResultSetMetaData rsmd
, final int tablePosition, @NotNull JSONObject table, final int columnIndex, Join join, Map<String, JSONObject> childMap) throws Exception {

if (table == null) { // 对应副表 viceSql 不能生成正常 SQL， 或者是 ! - Outer, ( - ANTI JOIN 的副表这种不需要缓存及返回的数据
Log.i(TAG, "onPutColumn table == null >> return table;");
return table;
}

if (isHideColumn(config, rs, rsmd, tablePosition, table, columnIndex, childMap)) {
Log.i(TAG, "onPutColumn isHideColumn(config, rs, rsmd, tablePosition, table, columnIndex, childMap) >> return table;");
return table;
}

//已改为 rsmd.getTableName(columnIndex) 支持副表不传 @column ， 但如何判断是副表？childMap != null
// String lable = rsmd.getColumnLabel(columnIndex);
// int dotIndex = lable.indexOf(".");
String lable = getKey(config, rs, rsmd, tablePosition, table, columnIndex, childMap);

// String childTable = childMap == null ? null : sqlTableName; // rsmd.getTableName(columnIndex); //dotIndex < 0 ? null : lable.substring(0, dotIndex);

JSONObject finalTable = null;
String childSql = null;

SQLConfig childConfig = join == null || join.isSQLJoin() == false ? null : join.getCacheConfig();
if (childConfig == null) {
finalTable = table;
}
else {
// lable = column;

//<sql, Table>

// List<Join> joinList = config.getJoinList();
// if (joinList != null) {
// for (Join j : joinList) {
// childConfig = j.isAppJoin() ? null : j.getCacheConfig(); //这里用config改了getSQL后再还原很麻烦，所以提前给一个config2更好

// FIXME 副表的 SQL 函数，甚至普通字段都可能从 rsmd.getTableName(columnIndex) 拿到 ""
// if (childConfig != null && childTable.equalsIgnoreCase(childConfig.getSQLTable())) {

childConfig.putWhere(join.getKey(), table.get(join.getTargetKey()), true);
childSql = childConfig.getSQL(false);

if (StringUtil.isEmpty(childSql, true)) {
return table;
}

finalTable = (JSONObject) childMap.get(childSql);
// break;
// }
// }
// }

}

Object value = getValue(config, rs, rsmd, tablePosition, table, columnIndex, lable, childMap);

// 必须 put 进去，否则某个字段为 null 可能导致中断后续正常返回值 if (value != null) {
if (finalTable == null) {
finalTable = new JSONObject(true);
childMap.put(childSql, finalTable);
}
finalTable.put(lable, value);
table.put(lable, value);
// }

return table;
Expand Down
7 changes: 4 additions & 3 deletions APIJSONORM/src/main/java/apijson/orm/Join.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,13 @@ public boolean isLeftOrRightJoin() {
String jt = getJoinType();
return "<".equals(jt) || ">".equals(jt);
}

public boolean canCacheViceTable() {
String jt = getJoinType();
return "@".equals(jt) || "<".equals(jt) || ">".equals(jt) || "&".equals(jt) || "*".equals(jt) || ")".equals(jt);
// 副表是按常规条件查询，缓存会导致其它同表同条件对象查询结果集为空 return ! isFullJoin(); // ! - OUTER, ( - FOREIGN 都需要缓存空副表数据，避免多余的查询
}

public boolean isSQLJoin() {
return ! isAppJoin();
}
Expand All @@ -215,7 +216,7 @@ public static boolean isSQLJoin(Join j) {
public static boolean isAppJoin(Join j) {
return j != null && j.isAppJoin();
}

public static boolean isLeftOrRightJoin(Join j) {
return j != null && j.isLeftOrRightJoin();
}
Expand Down

0 comments on commit 5d59b35

Please sign in to comment.