From d2083dd3952f4f59725f7b3220c0bc3bc0db2dca Mon Sep 17 00:00:00 2001 From: sabaimran Date: Sat, 1 Jul 2023 19:09:22 -0700 Subject: [PATCH] Remove bespoke processing for GithubToJsonl file demo --- src/khoj/processor/github/github_to_jsonl.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/khoj/processor/github/github_to_jsonl.py b/src/khoj/processor/github/github_to_jsonl.py index 7e0de284..537f9878 100644 --- a/src/khoj/processor/github/github_to_jsonl.py +++ b/src/khoj/processor/github/github_to_jsonl.py @@ -39,17 +39,13 @@ class GithubToJsonl(TextToJsonl): return def process(self, previous_entries=None): - # If demo mode is enabled, don't re-process any of the repositories. This is resource intensive. - if state.demo and previous_entries is not None: - return self.update_entries_with_ids(previous_entries, previous_entries) - current_entries = [] for repo in self.config.repos: - current_entries += self.process_repo(repo, previous_entries) + current_entries += self.process_repo(repo) return self.update_entries_with_ids(current_entries, previous_entries) - def process_repo(self, repo: GithubRepoConfig, previous_entries=None): + def process_repo(self, repo: GithubRepoConfig): repo_url = f"https://api.github.com/repos/{repo.owner}/{repo.name}" repo_shorthand = f"{repo.owner}/{repo.name}" logger.info(f"Processing github repo {repo_shorthand}")