Make PAT token requirement optional for Github indexing for now

The github integration has not been tested and may still be broken.
This change at least makes it easier to add repositories without
needing a PAT token if/when it does work.

src/khoj/interface/web/content_source_github_input.html

@@ -125,14 +125,6 @@
         event.preventDefault();
 
         const pat_token = document.getElementById("pat-token").value;
-
-        if (pat_token == "") {
-            document.getElementById("success").textContent = "❌ Please enter a Personal Access Token.";
-            document.getElementById("success").style.display = "block";
-            return;
-        }
-
-
         var cards = document.getElementById("repositories").getElementsByClassName("repo");
         var repos = [];
 

src/khoj/processor/content/github/github_to_entries.py

@@ -1,6 +1,6 @@
 import logging
 import time
-from typing import Any, Dict, List, Tuple
+from typing import Dict, List, Tuple
 
 import requests
 from magika import Magika
@@ -11,7 +11,7 @@ from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntrie
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.processor.content.text_to_entries import TextToEntries
-from khoj.utils.helpers import timer
+from khoj.utils.helpers import is_none_or_empty, timer
 from khoj.utils.rawconfig import GithubContentConfig, GithubRepoConfig
 
 logger = logging.getLogger(__name__)
@@ -36,6 +36,7 @@ class GithubToEntries(TextToEntries):
             repos=repos,
         )
         self.session = requests.Session()
+        if not is_none_or_empty(self.config.pat_token):
             self.session.headers.update({"Authorization": f"token {self.config.pat_token}"})
 
     @staticmethod
@@ -49,9 +50,10 @@ class GithubToEntries(TextToEntries):
             return
 
     def process(self, files: dict[str, str], user: KhojUser, regenerate: bool = False) -> Tuple[int, int]:
-        if self.config.pat_token is None or self.config.pat_token == "":
+        if is_none_or_empty(self.config.pat_token):
-            logger.error(f"Github PAT token is not set. Skipping github content")
+            logger.warning(
-            raise ValueError("Github PAT token is not set. Skipping github content")
+                f"Github PAT token is not set. Private repositories cannot be indexed and lower rate limits apply."
+            )
         current_entries = []
         for repo in self.config.repos:
             current_entries += self.process_repo(repo)
@@ -114,6 +116,8 @@ class GithubToEntries(TextToEntries):
     def get_files(self, repo_url: str, repo: GithubRepoConfig):
         # Get the contents of the repository
         repo_content_url = f"{repo_url}/git/trees/{repo.branch}"
+        headers = {}
+        if not is_none_or_empty(self.config.pat_token):
             headers = {"Authorization": f"token {self.config.pat_token}"}
         params = {"recursive": "true"}
         response = requests.get(repo_content_url, headers=headers, params=params)

src/khoj/utils/rawconfig.py

@@ -66,7 +66,7 @@ class GithubRepoConfig(ConfigBase):
 
 
 class GithubContentConfig(ConfigBase):
-    pat_token: str
+    pat_token: Optional[str] = None
     repos: List[GithubRepoConfig]