diff --git a/src/khoj/interface/web/assets/khoj.css b/src/khoj/interface/web/assets/khoj.css index 8ec85b10..1e2274a2 100644 --- a/src/khoj/interface/web/assets/khoj.css +++ b/src/khoj/interface/web/assets/khoj.css @@ -48,7 +48,7 @@ } .khoj-nav { display: grid; - grid-template-columns: repeat(3, 1fr); + grid-auto-flow: column; grid-gap: 32px; justify-self: center; } diff --git a/src/khoj/interface/web/base_config.html b/src/khoj/interface/web/base_config.html index 7e6bf09c..d15f63e4 100644 --- a/src/khoj/interface/web/base_config.html +++ b/src/khoj/interface/web/base_config.html @@ -2,6 +2,7 @@ + Khoj - Settings diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 56cc99d8..1ba00b20 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -113,13 +113,31 @@ } - +
- + {% if demo %} + +
+ + Check out what else we're building + +
+ {% endif %} + {% if demo %} + + + + {% else %} + + + + {% endif %}
@@ -296,5 +314,30 @@ grid-column: 2; } } + + div.khoj-banner-container { + background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107); + background-size: 400% 400%; + animation: gradient 15s ease infinite; + text-align: center; + padding: 10px; + } + + @keyframes gradient { + 0% { + background-position: 0% 50%; + } + 50% { + background-position: 100% 50%; + } + 100% { + background-position: 0% 50%; + } + } + + a.khoj-banner { + color: black; + text-decoration: none; + } diff --git a/src/khoj/interface/web/index.html b/src/khoj/interface/web/index.html index 49929b45..09e2e690 100644 --- a/src/khoj/interface/web/index.html +++ b/src/khoj/interface/web/index.html @@ -232,13 +232,36 @@ + {% if demo %} + +
+ +

+ Enroll in Khoj beta to get your own Github search engine +

+ + + +
+ {% endif %} +
- + {% if demo %} + + {% else %} + + {% endif %}
@@ -411,13 +434,90 @@ border-radius: 5px; padding: 10px; margin: 10px 0; - border: 1px solid rgb(229, 229, 229); + border: 4px solid rgb(229, 229, 229); } img { max-width: 90%; } + div.khoj-banner-container { + background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107); + background-size: 400% 400%; + animation: gradient 15s ease infinite; + text-align: center; + padding: 10px; + } + + @keyframes gradient { + 0% { + background-position: 0% 50%; + } + 50% { + background-position: 100% 50%; + } + 100% { + background-position: 0% 50%; + } + } + + a.khoj-banner { + color: black; + display: block; + } + + a.khoj-logo { + text-align: center; + } + + p.khoj-banner { + margin: 0; + padding: 10px; + } + + button#khoj-banner-submit, + input#khoj-banner-email { + padding: 10px; + border-radius: 5px; + border: 1px solid #475569; + background: #f9fafc; + } + + button#khoj-banner-submit:hover, + input#khoj-banner-email:hover { + box-shadow: 0 0 11px #aaa; + } + + diff --git a/src/khoj/main.py b/src/khoj/main.py index bfaec286..5c2c71c0 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -129,6 +129,7 @@ def set_state(args): state.verbose = args.verbose state.host = args.host state.port = args.port + state.demo = args.demo def start_server(app, host=None, port=None, socket=None): diff --git a/src/khoj/processor/github/github_to_jsonl.py b/src/khoj/processor/github/github_to_jsonl.py index 6f21749d..7e0de284 100644 --- a/src/khoj/processor/github/github_to_jsonl.py +++ b/src/khoj/processor/github/github_to_jsonl.py @@ -15,6 +15,7 @@ from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl from khoj.processor.text_to_jsonl import TextToJsonl from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data from khoj.utils.rawconfig import Entry +from khoj.utils import state logger = logging.getLogger(__name__) @@ -38,6 +39,10 @@ class GithubToJsonl(TextToJsonl): return def process(self, previous_entries=None): + # If demo mode is enabled, don't re-process any of the repositories. This is resource intensive. + if state.demo and previous_entries is not None: + return self.update_entries_with_ids(previous_entries, previous_entries) + current_entries = [] for repo in self.config.repos: current_entries += self.process_repo(repo, previous_entries) @@ -193,7 +198,7 @@ class GithubToJsonl(TextToJsonl): def _get_issues(self, issues_url: Union[str, None]) -> List[Dict]: issues = [] - per_page = 30 + per_page = 100 params = {"per_page": per_page, "state": "all"} while issues_url is not None: @@ -225,7 +230,7 @@ class GithubToJsonl(TextToJsonl): def get_comments(self, comments_url: Union[str, None]) -> List[Dict]: # By default, the number of results per page is 30. We'll keep it as-is for now. comments = [] - per_page = 30 + per_page = 100 params = {"per_page": per_page} while comments_url is not None: diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 58ba713d..0f57324b 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -39,6 +39,66 @@ from khoj.utils.yaml import save_config_to_file_updated_state api = APIRouter() logger = logging.getLogger(__name__) +if not state.demo: + + @api.get("/config/data", response_model=FullConfig) + def get_config_data(): + return state.config + + @api.post("/config/data") + async def set_config_data(updated_config: FullConfig): + state.config = updated_config + with open(state.config_file, "w") as outfile: + yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile) + outfile.close() + return state.config + + @api.post("/config/data/content_type/github", status_code=200) + async def set_content_config_github_data(updated_config: GithubContentConfig): + if not state.config: + state.config = FullConfig() + state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) + + if not state.config.content_type: + state.config.content_type = ContentConfig(**{"github": updated_config}) + else: + state.config.content_type.github = updated_config + + try: + save_config_to_file_updated_state() + return {"status": "ok"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @api.post("/config/data/content_type/{content_type}", status_code=200) + async def set_content_config_data(content_type: str, updated_config: TextContentConfig): + if not state.config: + state.config = FullConfig() + state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) + + if not state.config.content_type: + state.config.content_type = ContentConfig(**{content_type: updated_config}) + else: + state.config.content_type[content_type] = updated_config + + try: + save_config_to_file_updated_state() + return {"status": "ok"} + except Exception as e: + return {"status": "error", "message": str(e)} + + @api.post("/config/data/processor/conversation", status_code=200) + async def set_processor_conversation_config_data(updated_config: ConversationProcessorConfig): + if not state.config: + state.config = FullConfig() + state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) + state.config.processor = ProcessorConfig(conversation=updated_config) + try: + save_config_to_file_updated_state() + return {"status": "ok"} + except Exception as e: + return {"status": "error", "message": str(e)} + # Create Routes @api.get("/config/data/default") @@ -68,69 +128,6 @@ def get_config_types(): ] -@api.get("/config/data", response_model=FullConfig) -def get_config_data(): - return state.config - - -@api.post("/config/data") -async def set_config_data(updated_config: FullConfig): - state.config = updated_config - with open(state.config_file, "w") as outfile: - yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile) - outfile.close() - return state.config - - -@api.post("/config/data/content_type/github", status_code=200) -async def set_content_config_github_data(updated_config: GithubContentConfig): - if not state.config: - state.config = FullConfig() - state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) - - if not state.config.content_type: - state.config.content_type = ContentConfig(**{"github": updated_config}) - else: - state.config.content_type.github = updated_config - - try: - save_config_to_file_updated_state() - return {"status": "ok"} - except Exception as e: - return {"status": "error", "message": str(e)} - - -@api.post("/config/data/content_type/{content_type}", status_code=200) -async def set_content_config_data(content_type: str, updated_config: TextContentConfig): - if not state.config: - state.config = FullConfig() - state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) - - if not state.config.content_type: - state.config.content_type = ContentConfig(**{content_type: updated_config}) - else: - state.config.content_type[content_type] = updated_config - - try: - save_config_to_file_updated_state() - return {"status": "ok"} - except Exception as e: - return {"status": "error", "message": str(e)} - - -@api.post("/config/data/processor/conversation", status_code=200) -async def set_processor_conversation_config_data(updated_config: ConversationProcessorConfig): - if not state.config: - state.config = FullConfig() - state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"]) - state.config.processor = ProcessorConfig(conversation=updated_config) - try: - save_config_to_file_updated_state() - return {"status": "ok"} - except Exception as e: - return {"status": "error", "message": str(e)} - - @api.get("/search", response_model=List[SearchResponse]) async def search( q: str, diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 1ca57579..6ab94181 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -21,95 +21,94 @@ VALID_CONTENT_TYPES = ["org", "ledger", "markdown", "pdf"] # Create Routes @web_client.get("/", response_class=FileResponse) -def index(): - return FileResponse(constants.web_directory / "index.html") - - -@web_client.get("/config", response_class=HTMLResponse) -def config_page(request: Request): - return templates.TemplateResponse("config.html", context={"request": request}) - - -@web_client.get("/config/content_type/github", response_class=HTMLResponse) -def github_config_page(request: Request): - default_copy = constants.default_config.copy() - default_github = default_copy["content-type"]["github"] # type: ignore - - default_config = TextContentConfig( - compressed_jsonl=default_github["compressed-jsonl"], - embeddings_file=default_github["embeddings-file"], - ) - - current_config = ( - state.config.content_type.github - if state.config and state.config.content_type and state.config.content_type.github - else default_config - ) - - current_config = json.loads(current_config.json()) - - return templates.TemplateResponse( - "content_type_github_input.html", context={"request": request, "current_config": current_config} - ) - - -@web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse) -def content_config_page(request: Request, content_type: str): - if content_type not in VALID_CONTENT_TYPES: - return templates.TemplateResponse("config.html", context={"request": request}) - - default_copy = constants.default_config.copy() - default_content_type = default_copy["content-type"][content_type] # type: ignore - - default_config = TextContentConfig( - compressed_jsonl=default_content_type["compressed-jsonl"], - embeddings_file=default_content_type["embeddings-file"], - ) - - current_config = ( - state.config.content_type[content_type] - if state.config and state.config.content_type and state.config.content_type[content_type] # type: ignore - else default_config - ) - current_config = json.loads(current_config.json()) - - return templates.TemplateResponse( - "content_type_input.html", - context={ - "request": request, - "current_config": current_config, - "content_type": content_type, - }, - ) - - -@web_client.get("/config/processor/conversation", response_class=HTMLResponse) -def conversation_processor_config_page(request: Request): - default_copy = constants.default_config.copy() - default_processor_config = default_copy["processor"]["conversation"] # type: ignore - default_processor_config = ConversationProcessorConfig( - openai_api_key="", - model=default_processor_config["model"], - conversation_logfile=default_processor_config["conversation-logfile"], - chat_model=default_processor_config["chat-model"], - ) - - current_processor_conversation_config = ( - state.config.processor.conversation - if state.config and state.config.processor and state.config.processor.conversation - else default_processor_config - ) - current_processor_conversation_config = json.loads(current_processor_conversation_config.json()) - - return templates.TemplateResponse( - "processor_conversation_input.html", - context={ - "request": request, - "current_config": current_processor_conversation_config, - }, - ) +def index(request: Request): + return templates.TemplateResponse("index.html", context={"request": request, "demo": state.demo}) @web_client.get("/chat", response_class=FileResponse) -def chat_page(): - return FileResponse(constants.web_directory / "chat.html") +def chat_page(request: Request): + return templates.TemplateResponse("chat.html", context={"request": request, "demo": state.demo}) + + +if not state.demo: + + @web_client.get("/config", response_class=HTMLResponse) + def config_page(request: Request): + return templates.TemplateResponse("config.html", context={"request": request}) + + @web_client.get("/config/content_type/github", response_class=HTMLResponse) + def github_config_page(request: Request): + default_copy = constants.default_config.copy() + default_github = default_copy["content-type"]["github"] # type: ignore + + default_config = TextContentConfig( + compressed_jsonl=default_github["compressed-jsonl"], + embeddings_file=default_github["embeddings-file"], + ) + + current_config = ( + state.config.content_type.github + if state.config and state.config.content_type and state.config.content_type.github + else default_config + ) + + current_config = json.loads(current_config.json()) + + return templates.TemplateResponse( + "content_type_github_input.html", context={"request": request, "current_config": current_config} + ) + + @web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse) + def content_config_page(request: Request, content_type: str): + if content_type not in VALID_CONTENT_TYPES: + return templates.TemplateResponse("config.html", context={"request": request}) + + default_copy = constants.default_config.copy() + default_content_type = default_copy["content-type"][content_type] # type: ignore + + default_config = TextContentConfig( + compressed_jsonl=default_content_type["compressed-jsonl"], + embeddings_file=default_content_type["embeddings-file"], + ) + + current_config = ( + state.config.content_type[content_type] + if state.config and state.config.content_type and state.config.content_type[content_type] # type: ignore + else default_config + ) + current_config = json.loads(current_config.json()) + + return templates.TemplateResponse( + "content_type_input.html", + context={ + "request": request, + "current_config": current_config, + "content_type": content_type, + }, + ) + + @web_client.get("/config/processor/conversation", response_class=HTMLResponse) + def conversation_processor_config_page(request: Request): + default_copy = constants.default_config.copy() + default_processor_config = default_copy["processor"]["conversation"] # type: ignore + default_processor_config = ConversationProcessorConfig( + openai_api_key="", + model=default_processor_config["model"], + conversation_logfile=default_processor_config["conversation-logfile"], + chat_model=default_processor_config["chat-model"], + ) + + current_processor_conversation_config = ( + state.config.processor.conversation + if state.config and state.config.processor and state.config.processor.conversation + else default_processor_config + ) + current_processor_conversation_config = json.loads(current_processor_conversation_config.json()) + + return templates.TemplateResponse( + "processor_conversation_input.html", + context={ + "request": request, + "current_config": current_processor_conversation_config, + }, + ) diff --git a/src/khoj/utils/cli.py b/src/khoj/utils/cli.py index 37d65458..ba0ef971 100644 --- a/src/khoj/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -34,6 +34,7 @@ def cli(args=None): help="Path to UNIX socket for server. Use to run server behind reverse proxy. Default: /tmp/uvicorn.sock", ) parser.add_argument("--version", "-V", action="store_true", help="Print the installed Khoj version and exit") + parser.add_argument("--demo", action="store_true", default=False, help="Run Khoj in demo mode") args = parser.parse_args(args) diff --git a/src/khoj/utils/state.py b/src/khoj/utils/state.py index a3368084..d59239d8 100644 --- a/src/khoj/utils/state.py +++ b/src/khoj/utils/state.py @@ -27,6 +27,7 @@ search_index_lock = threading.Lock() SearchType = utils_config.SearchType telemetry: List[Dict[str, str]] = [] previous_query: str = None +demo: bool = False if torch.cuda.is_available(): # Use CUDA GPU