klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-06 13:22:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove bespoke processing for GithubToJsonl file demo

Browse Source
This commit is contained in:
sabaimran
2023-07-01 19:09:22 -07:00
parent a71440f62a
commit d2083dd395
1 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/khoj/processor/github/github_to_jsonl.py
View File

@@ -39,17 +39,13 @@ class GithubToJsonl(TextToJsonl):
return return
def process(self, previous_entries=None): def process(self, previous_entries=None):
# If demo mode is enabled, don't re-process any of the repositories. This is resource intensive.
if state.demo and previous_entries is not None:
return self.update_entries_with_ids(previous_entries, previous_entries)
current_entries = [] current_entries = []
for repo in self.config.repos: for repo in self.config.repos:
current_entries += self.process_repo(repo, previous_entries) current_entries += self.process_repo(repo)
return self.update_entries_with_ids(current_entries, previous_entries) return self.update_entries_with_ids(current_entries, previous_entries)
def process_repo(self, repo: GithubRepoConfig, previous_entries=None): def process_repo(self, repo: GithubRepoConfig):
repo_url = f"https://api.github.com/repos/{repo.owner}/{repo.name}" repo_url = f"https://api.github.com/repos/{repo.owner}/{repo.name}"
repo_shorthand = f"{repo.owner}/{repo.name}" repo_shorthand = f"{repo.owner}/{repo.name}"
logger.info(f"Processing github repo {repo_shorthand}") logger.info(f"Processing github repo {repo_shorthand}")