-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feat/seasearch add wiki search sup #366
Open
cir9no
wants to merge
6
commits into
master
Choose a base branch
from
feat/seasearch-add-wiki-search-sup
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
444663d
feat(search): add wiki search
cir9no 665b2b8
fix wiki index
cir9no 4ddbd6b
fix add wiki search
cir9no 7110744
chore: change some syntax
cir9no a684189
chore(wikisearch): compatible metadata code
cir9no 7e21984
fix repofilename index
cir9no File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,13 @@ | ||
from seafevents.seasearch.utils.constants import REPO_STATUS_FILENAME_INDEX_NAME | ||
|
||
|
||
class RepoStatus(object): | ||
def __init__(self, repo_id, from_commit, to_commit, metadata_updated_time): | ||
def __init__(self, repo_id, from_commit, to_commit, **kwargs): | ||
self.repo_id = repo_id | ||
self.from_commit = from_commit | ||
self.to_commit = to_commit | ||
self.metadata_updated_time = metadata_updated_time | ||
if 'metadata_updated_time' in kwargs: | ||
self.metadata_updated_time = kwargs['metadata_updated_time'] | ||
|
||
def need_recovery(self): | ||
return self.to_commit is not None | ||
|
@@ -35,9 +39,6 @@ class RepoStatusIndex(object): | |
'updatingto': { | ||
'type': 'keyword' | ||
}, | ||
'metadata_updated_time': { | ||
'type': 'keyword' | ||
}, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里把之前加的属性给删了,别的功能无法工作了 |
||
}, | ||
} | ||
|
||
|
@@ -46,8 +47,15 @@ def __init__(self, seasearch_api, index_name): | |
self.seasearch_api = seasearch_api | ||
self.create_index_if_missing() | ||
|
||
def is_status_filename_index(self): | ||
return self.index_name.startswith(REPO_STATUS_FILENAME_INDEX_NAME) | ||
|
||
def create_index_if_missing(self): | ||
if not self.seasearch_api.check_index_mapping(self.index_name).get('is_exist'): | ||
if self.is_status_filename_index(): | ||
self.mapping['properties']['metadata_updated_time'] = { | ||
'type': 'keyword' | ||
} | ||
data = { | ||
'mappings': self.mapping, | ||
} | ||
|
@@ -56,35 +64,45 @@ def create_index_if_missing(self): | |
def check_repo_status(self, repo_id): | ||
return self.seasearch_api.check_document_by_id(self.index_name, repo_id).get('is_exist') | ||
|
||
def add_repo_status(self, repo_id, commit_id, updatingto, metadata_updated_time): | ||
date = { | ||
def add_repo_status(self, repo_id, commit_id, updatingto, **kwargs): | ||
data = { | ||
'repo_id': repo_id, | ||
'commit_id': commit_id, | ||
'updatingto': updatingto, | ||
'metadata_updated_time': metadata_updated_time, | ||
} | ||
|
||
if 'metadata_updated_time' in kwargs: | ||
data.update(metadata_updated_time=kwargs['metadata_updated_time']) | ||
|
||
doc_id = repo_id | ||
self.seasearch_api.create_document_by_id(self.index_name, doc_id, date) | ||
self.seasearch_api.create_document_by_id(self.index_name, doc_id, data) | ||
|
||
def begin_update_repo(self, repo_id, old_commit_id, new_commit_id, metadata_updated_time): | ||
self.add_repo_status(repo_id, old_commit_id, new_commit_id, metadata_updated_time) | ||
def begin_update_repo(self, repo_id, old_commit_id, new_commit_id, **kwargs): | ||
self.add_repo_status(repo_id, old_commit_id, new_commit_id, **kwargs) | ||
|
||
def finish_update_repo(self, repo_id, commit_id, metadata_updated_time): | ||
self.add_repo_status(repo_id, commit_id, None, metadata_updated_time) | ||
def finish_update_repo(self, repo_id, commit_id, **kwargs): | ||
self.add_repo_status(repo_id, commit_id, None, **kwargs) | ||
|
||
def delete_documents_by_repo(self, repo_id): | ||
return self.seasearch_api.delete_document_by_id(self.index_name, repo_id) | ||
|
||
def get_repo_status_by_id(self, repo_id): | ||
doc = self.seasearch_api.get_document_by_id(self.index_name, repo_id) | ||
if doc.get('error'): | ||
return RepoStatus(repo_id, None, None, None) | ||
if self.is_status_filename_index(): | ||
return RepoStatus(repo_id, None, None, metadata_updated_time=None) | ||
else: | ||
return RepoStatus(repo_id, None, None) | ||
|
||
commit_id = doc['_source']['commit_id'] | ||
updatingto = doc['_source']['updatingto'] | ||
metadata_updated_time = doc['_source']['metadata_updated_time'] | ||
repo_id = doc['_source']['repo_id'] | ||
|
||
return RepoStatus(repo_id, commit_id, updatingto, metadata_updated_time) | ||
if self.is_status_filename_index(): | ||
metadata_updated_time = doc['_source']['metadata_updated_time'] | ||
return RepoStatus(repo_id, commit_id, updatingto, metadata_updated_time=metadata_updated_time) | ||
|
||
return RepoStatus(repo_id, commit_id, updatingto) | ||
|
||
def update_repo_status_by_id(self, doc_id, data): | ||
self.seasearch_api.update_document_by_id(self.index_name, doc_id, data) | ||
|
@@ -93,27 +111,28 @@ def get_repo_status_by_time(self, check_time): | |
per_size = 2000 | ||
start = 0 | ||
repo_head_list = [] | ||
while True: | ||
query_params = { | ||
"query": { | ||
"bool": { | ||
"must": [ | ||
{"range": | ||
{"@timestamp": | ||
{ | ||
"lt": check_time | ||
} | ||
query_params = { | ||
"query": { | ||
"bool": { | ||
"must": [ | ||
{"range": | ||
{"@timestamp": | ||
{ | ||
"lt": check_time | ||
} | ||
} | ||
] | ||
} | ||
}, | ||
"_source": ["commit_id", "updatingto", "metadata_updated_time"], | ||
"from": start, | ||
"size": per_size, | ||
"sort": ["-@timestamp"], | ||
} | ||
|
||
} | ||
] | ||
} | ||
}, | ||
"_source": ["commit_id", "updatingto"], | ||
"from": start, | ||
"size": per_size, | ||
"sort": ["-@timestamp"], | ||
} | ||
if self.is_status_filename_index(): | ||
query_params['_source'].append('metadata_updated_time') | ||
while True: | ||
repo_heads, total = self._repo_head_search(query_params) | ||
repo_head_list.extend(repo_heads) | ||
start += per_size | ||
|
@@ -151,13 +170,14 @@ def _repo_head_search(self, query_params): | |
repo_id = hit['_id'] | ||
commit_id = hit.get('_source').get('commit_id') | ||
updatingto = hit.get('_source').get('updatingto') | ||
metadata_updated_time = hit.get('_source').get('metadata_updated_time') | ||
repo_heads.append({ | ||
repo_head = { | ||
'repo_id': repo_id, | ||
'commit_id': commit_id, | ||
'updatingto': updatingto, | ||
'metadata_updated_time': metadata_updated_time, | ||
}) | ||
} | ||
if 'metadata_updated_time' in hit.get('_source', {}): | ||
repo_head['metadata_updated_time'] = hit.get('_source').get('metadata_updated_time') | ||
repo_heads.append(repo_head) | ||
return repo_heads, total | ||
|
||
def delete_index_by_index_name(self): | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
按照现在的逻辑这里还能正常recovery吗?