From 3314c8b0a4f8e4a4191067319457e6f3da628653 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 29 Aug 2024 11:10:48 +0800 Subject: [PATCH] [Fix](Variant) variant fallthrough with inverted index (#40069) When reading from segment, the schema type is variant, if we check type valid in `get_inverted_index`, the result should always return nullptr(since variant type it self does not support inverted index), but the actual storage could be `string` or etc.So we should ignore the type check and return the correct inverted index iterators introduced by #36163 --- be/src/olap/rowset/segment_v2/segment.cpp | 6 ++++-- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 7 ++++++- regression-test/data/variant_github_events_p2/load.out | 10 ++++++++++ .../suites/variant_github_events_p2/load.groovy | 9 +++++++-- .../suites/variant_p0/with_index/load.groovy | 1 + .../suites/variant_p0/with_index/var_index.groovy | 2 ++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index ac54c9252c95cb..64f58e546c22ba 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -203,7 +203,8 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o ColumnReader* reader = nullptr; if (col.is_extracted_column()) { auto relative_path = col.path_info_ptr()->copy_pop_front(); - const auto* node = _sub_column_tree[col.unique_id()].find_exact(relative_path); + int32_t unique_id = col.unique_id() > 0 ? col.unique_id() : col.parent_unique_id(); + const auto* node = _sub_column_tree[unique_id].find_exact(relative_path); reader = node != nullptr ? node->data.reader.get() : nullptr; } else { reader = _column_readers.contains(col.unique_id()) @@ -775,8 +776,9 @@ ColumnReader* Segment::_get_column_reader(const TabletColumn& col) { // init column iterator by path info if (col.has_path_info() || col.is_variant_type()) { auto relative_path = col.path_info_ptr()->copy_pop_front(); + int32_t unique_id = col.unique_id() > 0 ? col.unique_id() : col.parent_unique_id(); const auto* node = col.has_path_info() - ? _sub_column_tree[col.unique_id()].find_exact(relative_path) + ? _sub_column_tree[unique_id].find_exact(relative_path) : nullptr; if (node != nullptr) { return node->data.reader.get(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 958647a6bed977..db69be01882651 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1488,10 +1488,15 @@ Status SegmentIterator::_init_inverted_index_iterators() { } for (auto cid : _schema->column_ids()) { if (_inverted_index_iterators[cid] == nullptr) { + // Not check type valid, since we need to get inverted index for related variant type when reading the segment. + // If check type valid, we can not get inverted index for variant type, and result nullptr.The result for calling + // get_inverted_index with variant suffix should return corresponding inverted index meta. + bool check_inverted_index_by_type = false; // Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase. RETURN_IF_ERROR(_segment->new_inverted_index_iterator( _opts.tablet_schema->column(cid), - _segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid)), + _segment->_tablet_schema->get_inverted_index(_opts.tablet_schema->column(cid), + check_inverted_index_by_type), _opts, &_inverted_index_iterators[cid])); } } diff --git a/regression-test/data/variant_github_events_p2/load.out b/regression-test/data/variant_github_events_p2/load.out index 8d5e3327e3c12e..4bee99a71a99ea 100644 --- a/regression-test/data/variant_github_events_p2/load.out +++ b/regression-test/data/variant_github_events_p2/load.out @@ -11,3 +11,13 @@ 5451 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T02:48:28Z","id":"2489433218","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"created","comment":{"body":"Html5 도 같이 지원하는 업로더였으면 좋겠어요! 구글링 해보면 꽤 나와요 :)","created_at":"2015-01-01T02:48:27Z","html_url":"https://github.com/xpressengine/xe-core/issues/1086#issuecomment-68479093","id":68479093,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/68479093","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"issue":{"comments":4,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/comments","created_at":"2014-12-12T11:48:03Z","events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/events","html_url":"https://github.com/xpressengine/xe-core/issues/1086","id":51797879,"labels_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/labels{/name}","locked":0,"number":1086,"state":"open","title":"파일 업로더 교체","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","user":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?v=3","events_url":"https://api.github.com/users/bnu/events{/privacy}","followers_url":"https://api.github.com/users/bnu/followers","following_url":"https://api.github.com/users/bnu/following{/other_user}","gists_url":"https://api.github.com/users/bnu/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/bnu","id":53764,"login":"bnu","organizations_url":"https://api.github.com/users/bnu/orgs","received_events_url":"https://api.github.com/users/bnu/received_events","repos_url":"https://api.github.com/users/bnu/repos","site_admin":0,"starred_url":"https://api.github.com/users/bnu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bnu/subscriptions","type":"User","url":"https://api.github.com/users/bnu"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"IssueCommentEvent"} 5995 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T01:47:44Z","id":"2489414108","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"opened","number":1120,"pull_request":{"_links":{"comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments"},"commits":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits"},"html":{"href":"https://github.com/xpressengine/xe-core/pull/1120"},"issue":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120"},"review_comment":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}"},"review_comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments"},"self":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120"},"statuses":{"href":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654"}},"additions":4748,"base":{"label":"xpressengine:develop","ref":"develop","repo":{"archive_url":"https://api.github.com/repos/xpressengine/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/xpressengine/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/xpressengine/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/xpressengine/xe-core/branches{/branch}","clone_url":"https://github.com/xpressengine/xe-core.git","collaborators_url":"https://api.github.com/repos/xpressengine/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/xpressengine/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/xpressengine/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/xpressengine/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/xpressengine/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/xpressengine/xe-core/contributors","created_at":"2013-02-01T07:16:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/xpressengine/xe-core/downloads","events_url":"https://api.github.com/repos/xpressengine/xe-core/events","fork":0,"forks":143,"forks_count":143,"forks_url":"https://api.github.com/repos/xpressengine/xe-core/forks","full_name":"xpressengine/xe-core","git_commits_url":"https://api.github.com/repos/xpressengine/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/xpressengine/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/xpressengine/xe-core/git/tags{/sha}","git_url":"git://github.com/xpressengine/xe-core.git","has_downloads":1,"has_issues":1,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/xpressengine/xe-core/hooks","html_url":"https://github.com/xpressengine/xe-core","id":7953576,"issue_comment_url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/xpressengine/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/xpressengine/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/xpressengine/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/xpressengine/xe-core/languages","merges_url":"https://api.github.com/repos/xpressengine/xe-core/merges","milestones_url":"https://api.github.com/repos/xpressengine/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/xpressengine/xe-core/notifications{?since,all,participating}","open_issues":156,"open_issues_count":156,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"},"private":0,"pulls_url":"https://api.github.com/repos/xpressengine/xe-core/pulls{/number}","pushed_at":"2014-12-31T08:52:46Z","releases_url":"https://api.github.com/repos/xpressengine/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:xpressengine/xe-core.git","stargazers_count":149,"stargazers_url":"https://api.github.com/repos/xpressengine/xe-core/stargazers","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/xpressengine/xe-core/subscribers","subscription_url":"https://api.github.com/repos/xpressengine/xe-core/subscription","svn_url":"https://github.com/xpressengine/xe-core","tags_url":"https://api.github.com/repos/xpressengine/xe-core/tags","teams_url":"https://api.github.com/repos/xpressengine/xe-core/teams","trees_url":"https://api.github.com/repos/xpressengine/xe-core/git/trees{/sha}","updated_at":"2014-12-30T00:05:52Z","url":"https://api.github.com/repos/xpressengine/xe-core","watchers":149,"watchers_count":149},"sha":"c3430d1c724f42154ca5dd648637c4df796d1708","user":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"}},"body":"https://github.com/xpressengine/xe-core/issues/634\\r\\n\\r\\n# Internet Explorer send punycode URL(ASCII) URL and non-alphabet\\r\\nUnicode URL URL as a referer. 인터넷 익스플로러는 리퍼러 주소로 퓨니코드 주소와 유니코드 URL을 섞어\\r\\n쓰고 있습니다. AJAX 통신에는 리퍼러로 Unicode를 사용하고 요청 호스트로는 퓨니코드 URL을 사용(이건 다국어 주소\\r\\n형식으로 접속하려면 이렇게 했어야 할 것)합니다.\\r\\n- XE strictly compare referer and server host for denying CSRF, but\\r\\npunycode URL and Unicode URL should be dealt as a same one. 그런데 XE는 리퍼러의\\r\\n호스트와 서버 호스트를 비교합니다. punycode로 쓰인 주소와 Unicode로 쓰인 주소는 같은 주소를 지시하더라도 문자열이\\r\\n다릅니다. 같은 주소를 지칭하는 다른 문자열을 punycode로 변환해서 같은 주소라고 인식할 수 있게 수정했습니다.\\r\\n- Fix checkCSRF function to deal both form as a same one.\\r\\n- Convert Unicode URL input to punycode URL on the Admin Default URL\\r\\nSettings. 관리자가 유니코드 형식으로 기본 주소를 입력하더라도, 퓨니코드로 변환해 저장하도록 했습니다. 퓨니코드로 저장하는\\r\\n것이 여러모로 유용하기 때문입니다.\\r\\n- For converting punycode URL, include IDNA coverting class. 퓨니코드와 유니코드\\r\\n간 변환을 위해서 IDNA 변환 클래스(LGPL사용권)를 포함시켰습니다.\\r\\n\\r\\n**이 수정을 하면 *한글 도메인에서 글 작성이 가능*합니다. 하지만, *파일 업로드의 경우는 SWF Uploader 이슈로 파일 업로드가 불가능*합니다. 이 문제는, HTML5를 지원하는 파일 업로더를 이용하면 해결됩니다. (워드 프레스 등의 해결법) HTML5를 지원하는 파일 업로더는 AXISJ 업로더 등을 포함해서 XE 공식 홈페이지 자료실에서 다운받아 사용할 수 있습니다.(에디터 변경)**","changed_files":8,"comments":0,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments","commits":1,"commits_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits","created_at":"2015-01-01T01:47:43Z","deletions":1,"diff_url":"https://github.com/xpressengine/xe-core/pull/1120.diff","head":{"label":"misol:support_punycode_domain","ref":"support_punycode_domain","repo":{"archive_url":"https://api.github.com/repos/misol/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/misol/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/misol/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/misol/xe-core/branches{/branch}","clone_url":"https://github.com/misol/xe-core.git","collaborators_url":"https://api.github.com/repos/misol/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/misol/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/misol/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/misol/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/misol/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/misol/xe-core/contributors","created_at":"2014-12-31T14:41:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/misol/xe-core/downloads","events_url":"https://api.github.com/repos/misol/xe-core/events","fork":1,"forks":0,"forks_count":0,"forks_url":"https://api.github.com/repos/misol/xe-core/forks","full_name":"misol/xe-core","git_commits_url":"https://api.github.com/repos/misol/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/misol/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/misol/xe-core/git/tags{/sha}","git_url":"git://github.com/misol/xe-core.git","has_downloads":1,"has_issues":0,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/misol/xe-core/hooks","html_url":"https://github.com/misol/xe-core","id":28667946,"issue_comment_url":"https://api.github.com/repos/misol/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/misol/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/misol/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/misol/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/misol/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/misol/xe-core/languages","merges_url":"https://api.github.com/repos/misol/xe-core/merges","milestones_url":"https://api.github.com/repos/misol/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/misol/xe-core/notifications{?since,all,participating}","open_issues":0,"open_issues_count":0,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"},"private":0,"pulls_url":"https://api.github.com/repos/misol/xe-core/pulls{/number}","pushed_at":"2015-01-01T01:36:28Z","releases_url":"https://api.github.com/repos/misol/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:misol/xe-core.git","stargazers_count":0,"stargazers_url":"https://api.github.com/repos/misol/xe-core/stargazers","statuses_url":"https://api.github.com/repos/misol/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/misol/xe-core/subscribers","subscription_url":"https://api.github.com/repos/misol/xe-core/subscription","svn_url":"https://github.com/misol/xe-core","tags_url":"https://api.github.com/repos/misol/xe-core/tags","teams_url":"https://api.github.com/repos/misol/xe-core/teams","trees_url":"https://api.github.com/repos/misol/xe-core/git/trees{/sha}","updated_at":"2014-12-31T14:41:10Z","url":"https://api.github.com/repos/misol/xe-core","watchers":0,"watchers_count":0},"sha":"d2b05732abfd85020335ce272abd37c0ad1c6654","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"html_url":"https://github.com/xpressengine/xe-core/pull/1120","id":26739793,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120","locked":0,"mergeable_state":"unknown","merged":0,"number":1120,"patch_url":"https://github.com/xpressengine/xe-core/pull/1120.patch","review_comment_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}","review_comments":0,"review_comments_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments","state":"open","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654","title":"fix for Not-Alphabet URL document writing (#634)","updated_at":"2015-01-01T01:47:43Z","url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"PullRequestEvent"} +-- !sql -- +\N +\N +\N +\N +4748 + +-- !sql -- +135 + diff --git a/regression-test/suites/variant_github_events_p2/load.groovy b/regression-test/suites/variant_github_events_p2/load.groovy index 4e81dc2237c380..e1742231afc9ab 100644 --- a/regression-test/suites/variant_github_events_p2/load.groovy +++ b/regression-test/suites/variant_github_events_p2/load.groovy @@ -169,7 +169,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ // // build inverted index at middle of loading the data // ADD INDEX - sql """ ALTER TABLE github_events ADD INDEX idx_var (`v`) USING INVERTED PROPERTIES("parser" = "chinese", "parser_mode" = "fine_grained", "support_phrase" = "true") """ + sql """ ALTER TABLE github_events ADD INDEX idx_var (`v`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") """ wait_for_latest_op_on_table_finish("github_events", timeout) // 2022 @@ -214,7 +214,8 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ } while (running) } - + sql """set enable_match_without_inverted_index = false""" + // filter by bloom filter qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" qt_sql """select * from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1 limit 10""" sql """select * from github_events order by k limit 10""" @@ -230,4 +231,8 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ sql """insert into github_events2 select * from github_events order by k""" sql """select v['payload']['commits'] from github_events order by k ;""" sql """select v['payload']['commits'] from github_events2 order by k ;""" + + // query with inverted index + qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where v["repo"]["name"] match 'xpressengine' order by 1;""" + qt_sql """select count() from github_events where v["repo"]["name"] match 'apache' order by 1;""" } \ No newline at end of file diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index a5abbae2ab96ac..ba46e7a9eeed8b 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -61,6 +61,7 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ properties("replication_num" = "1", "disable_auto_compaction" = "true"); """ sql """insert into var_with_index values(1, '{"a" : 0, "b": 3}', 'hello world'), (2, '{"a" : 123}', 'world'),(3, '{"a" : 123}', 'hello world')""" + sql """set enable_match_without_inverted_index = false""" qt_sql_inv_1 """select v["a"] from var_with_index where inv match 'hello' order by k""" qt_sql_inv_2 """select v["a"] from var_with_index where inv match 'hello' and cast(v['a'] as int) > 0 order by k""" qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k""" diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index 5f061f3208bab8..a21d242954504a 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -33,7 +33,9 @@ suite("regression_test_variant_var_index", "p0"){ sql """insert into var_index values(2, '{"a" : 18811, "b" : "hello world", "c" : 1181111}')""" sql """insert into var_index values(3, '{"a" : 18811, "b" : "hello wworld", "c" : 11111}')""" sql """insert into var_index values(4, '{"a" : 1234, "b" : "hello xxx world", "c" : 8181111}')""" + sql """set enable_match_without_inverted_index = false""" qt_sql """select * from var_index where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" + sql """set enable_match_without_inverted_index = true""" sql """insert into var_index values(5, '{"a" : 123456789, "b" : 123456, "c" : 8181111}')""" qt_sql """select * from var_index where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" // insert double/float/array/json