diff --git a/languagetool-core/src/main/java/org/languagetool/Language.java b/languagetool-core/src/main/java/org/languagetool/Language.java
index 125abf36bdee7..564669aaaf92b 100644
--- a/languagetool-core/src/main/java/org/languagetool/Language.java
+++ b/languagetool-core/src/main/java/org/languagetool/Language.java
@@ -252,7 +252,13 @@ public List<Rule> getRelevantRemoteRules(ResourceBundle messageBundle, List<Remo
       .forEach(rules::add);
     rules.removeIf(rule -> {
       String activeRemoteRuleAbTest = ((RemoteRule) rule).getServiceConfiguration().getOptions().get("abtest"); //abtest option value must match the abtest value from server.properties
+      // allow disabling based on A/B test flags to compare multiple models
+      String excludeABTest = ((RemoteRule) rule).getServiceConfiguration().getOptions().get("excludeABTest");
       List<String> activeAbTestsForUser = userConfig.getAbTest();
+      if (excludeABTest != null && activeAbTestsForUser != null &&
+        activeAbTestsForUser.stream().anyMatch(flag -> flag.matches(excludeABTest))) {
+        return true;
+      }
       if (activeRemoteRuleAbTest != null && !activeRemoteRuleAbTest.trim().isEmpty()) { // A/B-Test active for remote rule
         if (activeAbTestsForUser == null) {
           return true; // No A/B-Tests are not active for user
diff --git a/languagetool-core/src/test/java/org/languagetool/rules/RemoteRuleTest.java b/languagetool-core/src/test/java/org/languagetool/rules/RemoteRuleTest.java
index ae2e8bb6133b1..077b6c13cac0d 100644
--- a/languagetool-core/src/test/java/org/languagetool/rules/RemoteRuleTest.java
+++ b/languagetool-core/src/test/java/org/languagetool/rules/RemoteRuleTest.java
@@ -28,16 +28,18 @@
 import org.junit.Test;
 import org.languagetool.AnalyzedSentence;
 import org.languagetool.JLanguageTool;
+import org.languagetool.UserConfig;
 import org.languagetool.language.Demo;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 public class RemoteRuleTest {
 
@@ -173,4 +175,48 @@ public void testFailedRequests() throws IOException {
     assertMatches("no matches for failing requests", 0);
   }
 
+  private UserConfig getUserConfigWithAbTest(List<String> abTest) {
+    UserConfig userConfig = new UserConfig(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(),
+      5, null, null, null, null, false, abTest, null, false, null);
+    return userConfig;
+  }
+
+  @Test
+  public void testAbFlags() throws IOException {
+    lt = new JLanguageTool(new Demo());
+    assertFalse(lt.getAllActiveRules().stream().anyMatch(r -> r.getId().equals(config.ruleId)));
+
+    RemoteRuleConfig c1 = new RemoteRuleConfig(config);
+    c1.options.put("abtest", "foo");
+    lt = new JLanguageTool(new Demo());
+    lt.activateRemoteRules(Arrays.asList(c1));
+
+    assertFalse(lt.getAllActiveRules().stream().anyMatch(r -> r.getId().equals(config.ruleId)));
+
+    UserConfig u1 = getUserConfigWithAbTest(Arrays.asList("foo"));
+    lt = new JLanguageTool(new Demo(), null, u1);
+    lt.activateRemoteRules(Arrays.asList(c1));
+
+    assertTrue(lt.getAllActiveRules().stream().anyMatch(r -> r.getId().equals(config.ruleId)));
+
+    UserConfig u2 = getUserConfigWithAbTest(Arrays.asList("foo", "bar"));
+    RemoteRuleConfig c2 = new RemoteRuleConfig(config);
+    c2.options.put("abtest", "bar");
+    c2.options.put("excludeABTest", "fo.");
+    lt = new JLanguageTool(new Demo(), null, u2);
+    lt.activateRemoteRules(Arrays.asList(c2));
+
+    assertFalse(lt.getAllActiveRules().stream().anyMatch(r -> r.getId().equals(config.ruleId)));
+
+    UserConfig u3 = getUserConfigWithAbTest(Arrays.asList("bar"));
+    lt = new JLanguageTool(new Demo(), null, u3);
+    lt.activateRemoteRules(Arrays.asList(c2));
+
+    assertTrue(lt.getAllActiveRules().stream().anyMatch(r -> r.getId().equals(config.ruleId)));
+
+    RemoteRuleConfig c3 = new RemoteRuleConfig(config);
+    UserConfig u4 = getUserConfigWithAbTest(Arrays.asList());
+    c2.options.put("excludeABTest", "fo.");
+  }
+
 }