Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Legend SQL - pattern matching operator support #2

Open
wants to merge 8 commits into
base: finos-master
Choose a base branch
from
Open

Legend SQL - pattern matching operator support #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
63ec281
Legend SQL - support pattern matching operators
gs-jp1 Mar 27, 2024
bca1939
Remove overrideFlag from freemarker processing (#2724)
sprisha Apr 1, 2024
caf4623
Remove executionPlan dependency on xts-authentication (#2743)
pierredebelen Apr 1, 2024
bca2018
Remove tests that duplicate legend-pure tests (#2741)
kevin-m-knight-gs Apr 2, 2024
3d4c9a4
Add interface method (#2742)
Yasirmod17 Apr 2, 2024
fd619d4
Improve usage of removeDuplicates (#2740)
kevin-m-knight-gs Apr 2, 2024
5a71dc0
Merge pull request #7 from goldmansachs/review-jp-sql-18
gs-jp1 Apr 2, 2024
622fa7c
Fix lint
gs-jp1 Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions ...legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutor;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorBuilder;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorConfiguration;
import org.finos.legend.engine.plan.execution.stores.inMemory.plugin.InMemory;
import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutor;
import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutorBuilder;
Expand Down Expand Up @@ -283,9 +284,10 @@ public void run(T serverConfiguration, Environment environment)
ServiceStoreExecutor serviceStoreExecutor = (ServiceStoreExecutor) new ServiceStoreExecutorBuilder().build(serviceStoreExecutionConfiguration);

MongoDBStoreExecutorConfiguration mongoDBExecutorConfiguration = MongoDBStoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build();
MongoDBStoreExecutor mongoDBStoreExecutor = (MongoDBStoreExecutor) new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration);
MongoDBStoreExecutor mongoDBStoreExecutor = new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration);

ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build();
ElasticsearchV7StoreExecutorConfiguration elasticsearchV7StoreExecutorConfiguration = ElasticsearchV7StoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build();
ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build(elasticsearchV7StoreExecutorConfiguration);

PlanExecutor planExecutor;
ParallelGraphFetchExecutionExecutorPool parallelGraphFetchExecutionExecutorPool = null;
Expand Down
4 changes: 0 additions & 4 deletions .../legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,6 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-dependencies</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-authentication-implementation-core</artifactId>
</dependency>
<!-- ENGINE -->

<!-- COMMONS LANG -->
Expand Down
16 changes: 3 additions & 13 deletions ...tionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/PlanExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.utility.Iterate;
import org.eclipse.collections.impl.utility.internal.IterableIterate;
import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider;
import org.finos.legend.engine.plan.execution.concurrent.ConcurrentExecutionNodeExecutorPool;
import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool;
import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration;
Expand Down Expand Up @@ -78,17 +77,15 @@ public class PlanExecutor
private ParallelGraphFetchExecutionExecutorPool graphFetchExecutionNodeExecutorPool;
private GraphFetchExecutionConfiguration graphFetchExecutionConfiguration;
private BiFunction<Identity, ExecutionState, ExecutionNodeExecutor> executionNodeExecutorBuilder;
private final CredentialProviderProvider credentialProviderProvider;
private final boolean logSQLWithParamValues;


private PlanExecutor(boolean isJavaCompilationAllowed, ImmutableList<StoreExecutor> extraExecutors, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
private PlanExecutor(boolean isJavaCompilationAllowed, ImmutableList<StoreExecutor> extraExecutors, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
{
EngineUrlStreamHandlerFactory.initialize();
this.isJavaCompilationAllowed = isJavaCompilationAllowed;
this.extraExecutors = extraExecutors;
this.planExecutorInfo = PlanExecutorInfo.fromStoreExecutors(this.extraExecutors);
this.credentialProviderProvider = credentialProviderProvider;
this.graphFetchExecutionConfiguration = graphFetchExecutionConfiguration;
this.logSQLWithParamValues = logSQLWithParamValues;
}
Expand Down Expand Up @@ -373,7 +370,7 @@ public ExecutionState buildDefaultExecutionState(SingleExecutionPlan executionPl

private ExecutionState buildDefaultExecutionState(SingleExecutionPlan executionPlan, Map<String, Result> vars, PlanExecutionContext planExecutionContext)
{
ExecutionState executionState = new ExecutionState(vars, executionPlan.templateFunctions, this.extraExecutors.collect(StoreExecutor::buildStoreExecutionState), this.isJavaCompilationAllowed, null, this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
ExecutionState executionState = new ExecutionState(vars, executionPlan.templateFunctions, this.extraExecutors.collect(StoreExecutor::buildStoreExecutionState), this.isJavaCompilationAllowed, null, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);

if (planExecutionContext != null)
{
Expand Down Expand Up @@ -435,7 +432,6 @@ public static class Builder
private boolean isJavaCompilationAllowed = DEFAULT_IS_JAVA_COMPILATION_ALLOWED;
private final MutableList<StoreExecutor> storeExecutors = Lists.mutable.empty();
private GraphFetchExecutionConfiguration graphFetchExecutionConfiguration = new GraphFetchExecutionConfiguration();
private CredentialProviderProvider credentialProviderProvider = CredentialProviderProvider.defaultProviderProvider();
private boolean logSQLWithParamValues = true;

private Builder()
Expand Down Expand Up @@ -474,12 +470,6 @@ public Builder withAvailableStoreExecutors()
return this;
}

public Builder withCredentialProviderProvider(CredentialProviderProvider credentialProviderProvider)
{
this.credentialProviderProvider = credentialProviderProvider;
return this;
}

public Builder logSQLWithParamValues(boolean value)
{
this.logSQLWithParamValues = value;
Expand All @@ -488,7 +478,7 @@ public Builder logSQLWithParamValues(boolean value)

public PlanExecutor build()
{
return new PlanExecutor(this.isJavaCompilationAllowed, this.storeExecutors.toImmutable(), this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
return new PlanExecutor(this.isJavaCompilationAllowed, this.storeExecutors.toImmutable(), this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
}
}

Expand Down
10 changes: 1 addition & 9 deletions ...a/org/finos/legend/engine/plan/execution/nodes/helpers/freemarker/FreeMarkerExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,7 @@ public boolean isEmpty()

public static String processRecursively(String input, Map<String, ?> variableMap, String templateFunctions)
{
boolean overrideTemplateModelFlag = Boolean.valueOf(System.getProperty(overridePropertyForTemplateModel));
if (!overrideTemplateModelFlag)
{
return process(input, new TemplateHashModelOverride(variableMap, templateFunctions), templateFunctions);
}
else
{
return recur(input, variableMap, templateFunctions);
}
return process(input, new TemplateHashModelOverride(variableMap, templateFunctions), templateFunctions);
}

private static String recur(String input, Map<String,?> variableMap, String templateFunctions)
Expand Down
33 changes: 12 additions & 21 deletions ...tion/src/main/java/org/finos/legend/engine/plan/execution/nodes/state/ExecutionState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider;
import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool;
import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration;
import org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache;
Expand Down Expand Up @@ -80,7 +79,6 @@ public class ExecutionState
public final List<Function3<ExecutionNode,Identity, ExecutionState, Result>> extraNodeExecutors;
public final List<Function3<ExecutionNode, Identity, ExecutionState, Result>> extraSequenceNodeExecutors;
public RequestContext requestContext;
private final CredentialProviderProvider credentialProviderProvider;

public ExecutionState(ExecutionState state)
{
Expand Down Expand Up @@ -108,44 +106,43 @@ public ExecutionState(ExecutionState state)
this.extraNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraNodeExecutors);
this.extraSequenceNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraSequenceNodeExecutors);
this.requestContext = state.requestContext;
this.credentialProviderProvider = state.credentialProviderProvider;
this.logSQLWithParamValues = state.logSQLWithParamValues;
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, new RequestContext(), null);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, new RequestContext());
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, new RequestContext(), null, graphFetchExecutionConfiguration);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, new RequestContext(), graphFetchExecutionConfiguration);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, new GraphFetchExecutionConfiguration());
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, new GraphFetchExecutionConfiguration());
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, requestContext, credentialProviderProvider, true);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, requestContext, true);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, graphFetchExecutionConfiguration, true);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, graphFetchExecutionConfiguration, true);
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, boolean logSQLWithParamValues)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, boolean logSQLWithParamValues)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, new GraphFetchExecutionConfiguration(graphFetchBatchMemoryLimit), logSQLWithParamValues);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, new GraphFetchExecutionConfiguration(graphFetchBatchMemoryLimit), logSQLWithParamValues);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
{
this.inAllocation = false;
this.inLake = false;
Expand All @@ -160,7 +157,6 @@ public ExecutionState(Map<String, Result> res, List<? extends String> templateFu
this.extraNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraNodeExecutors);
this.extraSequenceNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraSequenceNodeExecutors);
this.requestContext = requestContext;
this.credentialProviderProvider = credentialProviderProvider;
this.logSQLWithParamValues = logSQLWithParamValues;
}

Expand All @@ -179,7 +175,7 @@ public ExecutionState copy()
Map<String, Result> resCopy = Maps.mutable.ofMap(this.res);
List<? extends String> templateFunctionsCopy = Lists.mutable.ofAll(this.templateFunctions);
List<? extends StoreExecutionState> extraStatesCopy = this.states.values().stream().map(StoreExecutionState::copy).collect(Collectors.toList());
ExecutionState copy = new ExecutionState(resCopy, templateFunctionsCopy, extraStatesCopy, this.isJavaCompilationAllowed, this.requestContext, this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
ExecutionState copy = new ExecutionState(resCopy, templateFunctionsCopy, extraStatesCopy, this.isJavaCompilationAllowed, this.requestContext, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);

copy.activities = Lists.mutable.withAll(this.activities);
copy.allocationNodeName = this.allocationNodeName;
Expand Down Expand Up @@ -358,11 +354,6 @@ public void setRequestContext(RequestContext requestContext)
this.requestContext = requestContext;
}

public CredentialProviderProvider getCredentialProviderProvider()
{
return this.credentialProviderProvider;
}

public boolean logSQLWithParamValues()
{
return this.logSQLWithParamValues;
Expand Down
6 changes: 3 additions & 3 deletions ...Plan-execution/src/test/java/org/finos/legend/engine/plan/execution/TestPlanExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,17 @@ public void testStateWithReferralHeaderPlaceHolder()
res.put("userId", new ConstantResult("anumam"));
Identity identity = IdentityFactoryProvider.getInstance().getAnonymousIdentity();

ExecutionState state1 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, "https://allo'y'.site.gs.com/"), null);
ExecutionState state1 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, "https://allo'y'.site.gs.com/"));
PlanExecutor.setUpState(new SingleExecutionPlan(), state1, identity, "anumam");
String sqlQuery1 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state1, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"https://allo''y''.site.gs.com/\"}';", sqlQuery1);

ExecutionState state2 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, null), null);
ExecutionState state2 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, null));
PlanExecutor.setUpState(new SingleExecutionPlan(), state2, identity, "anumam");
String sqlQuery2 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state2, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"null\"}';", sqlQuery2);

ExecutionState state3 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, null, null);
ExecutionState state3 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, null);
PlanExecutor.setUpState(new SingleExecutionPlan(), state3, identity, "anumam");
String sqlQuery3 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state3, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"null\"}';", sqlQuery3);
Expand Down
3 changes: 0 additions & 3 deletions ...g/finos/legend/engine/plan/execution/nodes/helpers/freemarker/TestFreeMarkerExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,6 @@ public void testFreemarkerStringWithCombinationsAndSpecialCharacters() throws Ex
rootMap.put("C3", "${C4}");
rootMap.put("C4", "C4");
rootMap.put("D", "abcd<@efg");
//should fail in the old flow (since old flow processes placeholder with specialcharacter value twice, causing it to fail
Assert.assertThrows(RuntimeException.class, () -> processRecursivelyWithFlagSwitching(sql, rootMap, "", "this is A1 and B2 and C4 placeholders and abcd<@efg with special characters."));
System.clearProperty(overridePropertyForTemplateModel);
//processing should pass with new flow since it allows placeholder to process only once and avoid this issue.
Assert.assertEquals("this is A1 and B2 and C4 placeholders and abcd<@efg with special characters.", processRecursively(sql, rootMap, ""));
}
Expand Down
Loading