klbr/khoj
1
0
Fork 0
mirror of https://github.com/khoaliber/khoj.git synced 2026-03-08 05:39:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add separate configuration if the given Khoj instance is meant for demo

Browse Source 
- In theory, this will be suitable for any Khoj instance that's meant for external-facing purposes (as in, outside of the user's network)
- Prevent re-indexing for Github data if this is a demo instance
- Fix up some issues with the CSS which made settings page small in mobile
- In the frontend views for Khoj, add a button to get on the waitlist and links to the landing page
This commit is contained in:
sabaimran
2023-06-30 20:38:55 -07:00
parent db3026739d
commit 796102c74e
10 changed files with 309 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/khoj/interface/web/assets/khoj.css
View File

@@ -48,7 +48,7 @@
} }
.khoj-nav { .khoj-nav {
display: grid; display: grid;
grid-template-columns: repeat(3, 1fr); grid-auto-flow: column;
grid-gap: 32px; grid-gap: 32px;
justify-self: center; justify-self: center;
} }

1
src/khoj/interface/web/base_config.html
View File

@@ -2,6 +2,7 @@
<html data-theme="light"> <html data-theme="light">
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png"> <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
<title>Khoj - Settings</title> <title>Khoj - Settings</title>
<link rel="stylesheet" href="/static/assets/pico.min.css"> <link rel="stylesheet" href="/static/assets/pico.min.css">

43
src/khoj/interface/web/chat.html
View File

@@ -115,11 +115,29 @@
<body> <body>
<!--Add Header Logo and Nav Pane--> <!--Add Header Logo and Nav Pane-->
<div class="khoj-header"> <div class="khoj-header">
{% if demo %}
<!-- Banner linking to https://khoj.dev -->
<div class="khoj-banner-container">
<a class="khoj-banner" href="https://khoj.dev" target="_blank">
Check out what else we're building
</a>
</div>
{% endif %}
{% if demo %}
<a href="https://khoj.dev" target="_blank">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img> <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img>
</a>
{% else %}
<a href="https://lantern.khoj.dev" target="_blank">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img>
</a>
{% endif %}
<nav class="khoj-nav"> <nav class="khoj-nav">
<a class="khoj-nav-selected" href="/chat">Chat</a> <a class="khoj-nav-selected" href="/chat">Chat</a>
<a href="/">Search</a> <a href="/">Search</a>
{% if not demo %}
<a href="/config">Settings</a> <a href="/config">Settings</a>
{% endif %}
</nav> </nav>
</div> </div>
@@ -296,5 +314,30 @@
grid-column: 2; grid-column: 2;
} }
} }
div.khoj-banner-container {
background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107);
background-size: 400% 400%;
animation: gradient 15s ease infinite;
text-align: center;
padding: 10px;
}
@keyframes gradient {
0% {
background-position: 0% 50%;
}
50% {
background-position: 100% 50%;
}
100% {
background-position: 0% 50%;
}
}
a.khoj-banner {
color: black;
text-decoration: none;
}
</style> </style>
</html> </html>

102
src/khoj/interface/web/index.html
View File

@@ -232,13 +232,36 @@
</script> </script>
<body> <body>
{% if demo %}
<!-- Banner linking to https://khoj.dev -->
<div class="khoj-banner-container">
<a class="khoj-banner" href="https://khoj.dev" target="_blank">
<p id="khoj-banner" class="khoj-banner">
Enroll in Khoj beta to get your own Github search engine
</p>
</a>
<input type="text" id="khoj-banner-email" placeholder="email" class="khoj-banner-email"></input>
<button id="khoj-banner-submit" class="khoj-banner-button">Submit</button>
</div>
{% endif %}
<!--Add Header Logo and Nav Pane--> <!--Add Header Logo and Nav Pane-->
<div class="khoj-header"> <div class="khoj-header">
{% if demo %}
<a class="khoj-logo" href="https://khoj.dev" target="_blank">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img> <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img>
</a>
{% else %}
<a class="khoj-logo" href="https://lantern.khoj.dev" target="_blank">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways.svg" alt="Khoj"></img>
</a>
{% endif %}
<nav class="khoj-nav"> <nav class="khoj-nav">
<a href="/chat">Chat</a> <a href="/chat">Chat</a>
<a class="khoj-nav-selected" href="/">Search</a> <a class="khoj-nav-selected" href="/">Search</a>
{% if not demo %}
<a href="/config">Settings</a> <a href="/config">Settings</a>
{% endif %}
</nav> </nav>
</div> </div>
@@ -411,13 +434,90 @@
border-radius: 5px; border-radius: 5px;
padding: 10px; padding: 10px;
margin: 10px 0; margin: 10px 0;
border: 1px solid rgb(229, 229, 229); border: 4px solid rgb(229, 229, 229);
} }
img { img {
max-width: 90%; max-width: 90%;
} }
div.khoj-banner-container {
background: linear-gradient(-45deg, #FFC107, #FF9800, #FF5722, #FF9800, #FFC107);
background-size: 400% 400%;
animation: gradient 15s ease infinite;
text-align: center;
padding: 10px;
}
@keyframes gradient {
0% {
background-position: 0% 50%;
}
50% {
background-position: 100% 50%;
}
100% {
background-position: 0% 50%;
}
}
a.khoj-banner {
color: black;
display: block;
}
a.khoj-logo {
text-align: center;
}
p.khoj-banner {
margin: 0;
padding: 10px;
}
button#khoj-banner-submit,
input#khoj-banner-email {
padding: 10px;
border-radius: 5px;
border: 1px solid #475569;
background: #f9fafc;
}
button#khoj-banner-submit:hover,
input#khoj-banner-email:hover {
box-shadow: 0 0 11px #aaa;
}
</style> </style>
<script>
var khojBannerSubmit = document.getElementById("khoj-banner-submit");
khojBannerSubmit.addEventListener("click", function(event) {
event.preventDefault();
var email = document.getElementById("khoj-banner-email").value;
fetch("https://lantern.khoj.dev/beta/users/", {
method: "POST",
body: JSON.stringify({
email: email
}),
headers: {
"Content-Type": "application/json"
}
}).then(function(response) {
return response.json();
}).then(function(data) {
console.log(data);
if (data.user != null) {
document.getElementById("khoj-banner").innerHTML = "Thanks for signing up. We'll be in touch soon! 🚀";
document.getElementById("khoj-banner-submit").remove();
} else {
document.getElementById("khoj-banner").innerHTML = "There was an error signing up. Please contact team@khoj.dev";
}
}).catch(function(error) {
console.log(error);
document.getElementById("khoj-banner").innerHTML = "There was an error signing up. Please contact team@khoj.dev";
});
});
</script>
</html> </html>

1
src/khoj/main.py
View File

@@ -129,6 +129,7 @@ def set_state(args):
state.verbose = args.verbose state.verbose = args.verbose
state.host = args.host state.host = args.host
state.port = args.port state.port = args.port
state.demo = args.demo
def start_server(app, host=None, port=None, socket=None): def start_server(app, host=None, port=None, socket=None):

9
src/khoj/processor/github/github_to_jsonl.py
View File

@@ -15,6 +15,7 @@ from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
from khoj.processor.text_to_jsonl import TextToJsonl from khoj.processor.text_to_jsonl import TextToJsonl
from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
from khoj.utils.rawconfig import Entry from khoj.utils.rawconfig import Entry
from khoj.utils import state
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -38,6 +39,10 @@ class GithubToJsonl(TextToJsonl):
return return
def process(self, previous_entries=None): def process(self, previous_entries=None):
# If demo mode is enabled, don't re-process any of the repositories. This is resource intensive.
if state.demo and previous_entries is not None:
return self.update_entries_with_ids(previous_entries, previous_entries)
current_entries = [] current_entries = []
for repo in self.config.repos: for repo in self.config.repos:
current_entries += self.process_repo(repo, previous_entries) current_entries += self.process_repo(repo, previous_entries)
@@ -193,7 +198,7 @@ class GithubToJsonl(TextToJsonl):
def _get_issues(self, issues_url: Union[str, None]) -> List[Dict]: def _get_issues(self, issues_url: Union[str, None]) -> List[Dict]:
issues = [] issues = []
per_page = 30 per_page = 100
params = {"per_page": per_page, "state": "all"} params = {"per_page": per_page, "state": "all"}
while issues_url is not None: while issues_url is not None:
@@ -225,7 +230,7 @@ class GithubToJsonl(TextToJsonl):
def get_comments(self, comments_url: Union[str, None]) -> List[Dict]: def get_comments(self, comments_url: Union[str, None]) -> List[Dict]:
# By default, the number of results per page is 30. We'll keep it as-is for now. # By default, the number of results per page is 30. We'll keep it as-is for now.
comments = [] comments = []
per_page = 30 per_page = 100
params = {"per_page": per_page} params = {"per_page": per_page}
while comments_url is not None: while comments_url is not None:

123
src/khoj/routers/api.py
View File

@@ -39,6 +39,66 @@ from khoj.utils.yaml import save_config_to_file_updated_state
api = APIRouter() api = APIRouter()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if not state.demo:
@api.get("/config/data", response_model=FullConfig)
def get_config_data():
return state.config
@api.post("/config/data")
async def set_config_data(updated_config: FullConfig):
state.config = updated_config
with open(state.config_file, "w") as outfile:
yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile)
outfile.close()
return state.config
@api.post("/config/data/content_type/github", status_code=200)
async def set_content_config_github_data(updated_config: GithubContentConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
if not state.config.content_type:
state.config.content_type = ContentConfig(**{"github": updated_config})
else:
state.config.content_type.github = updated_config
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
@api.post("/config/data/content_type/{content_type}", status_code=200)
async def set_content_config_data(content_type: str, updated_config: TextContentConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
if not state.config.content_type:
state.config.content_type = ContentConfig(**{content_type: updated_config})
else:
state.config.content_type[content_type] = updated_config
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
@api.post("/config/data/processor/conversation", status_code=200)
async def set_processor_conversation_config_data(updated_config: ConversationProcessorConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
state.config.processor = ProcessorConfig(conversation=updated_config)
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
# Create Routes # Create Routes
@api.get("/config/data/default") @api.get("/config/data/default")
@@ -68,69 +128,6 @@ def get_config_types():
] ]
@api.get("/config/data", response_model=FullConfig)
def get_config_data():
return state.config
@api.post("/config/data")
async def set_config_data(updated_config: FullConfig):
state.config = updated_config
with open(state.config_file, "w") as outfile:
yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile)
outfile.close()
return state.config
@api.post("/config/data/content_type/github", status_code=200)
async def set_content_config_github_data(updated_config: GithubContentConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
if not state.config.content_type:
state.config.content_type = ContentConfig(**{"github": updated_config})
else:
state.config.content_type.github = updated_config
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
@api.post("/config/data/content_type/{content_type}", status_code=200)
async def set_content_config_data(content_type: str, updated_config: TextContentConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
if not state.config.content_type:
state.config.content_type = ContentConfig(**{content_type: updated_config})
else:
state.config.content_type[content_type] = updated_config
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
@api.post("/config/data/processor/conversation", status_code=200)
async def set_processor_conversation_config_data(updated_config: ConversationProcessorConfig):
if not state.config:
state.config = FullConfig()
state.config.search_type = SearchConfig.parse_obj(constants.default_config["search-type"])
state.config.processor = ProcessorConfig(conversation=updated_config)
try:
save_config_to_file_updated_state()
return {"status": "ok"}
except Exception as e:
return {"status": "error", "message": str(e)}
@api.get("/search", response_model=List[SearchResponse]) @api.get("/search", response_model=List[SearchResponse])
async def search( async def search(
q: str, q: str,

35
src/khoj/routers/web_client.py
View File

@@ -21,17 +21,23 @@ VALID_CONTENT_TYPES = ["org", "ledger", "markdown", "pdf"]
# Create Routes # Create Routes
@web_client.get("/", response_class=FileResponse) @web_client.get("/", response_class=FileResponse)
def index(): def index(request: Request):
return FileResponse(constants.web_directory / "index.html") return templates.TemplateResponse("index.html", context={"request": request, "demo": state.demo})
@web_client.get("/config", response_class=HTMLResponse) @web_client.get("/chat", response_class=FileResponse)
def config_page(request: Request): def chat_page(request: Request):
return templates.TemplateResponse("chat.html", context={"request": request, "demo": state.demo})
if not state.demo:
@web_client.get("/config", response_class=HTMLResponse)
def config_page(request: Request):
return templates.TemplateResponse("config.html", context={"request": request}) return templates.TemplateResponse("config.html", context={"request": request})
@web_client.get("/config/content_type/github", response_class=HTMLResponse)
@web_client.get("/config/content_type/github", response_class=HTMLResponse) def github_config_page(request: Request):
def github_config_page(request: Request):
default_copy = constants.default_config.copy() default_copy = constants.default_config.copy()
default_github = default_copy["content-type"]["github"] # type: ignore default_github = default_copy["content-type"]["github"] # type: ignore
@@ -52,9 +58,8 @@ def github_config_page(request: Request):
"content_type_github_input.html", context={"request": request, "current_config": current_config} "content_type_github_input.html", context={"request": request, "current_config": current_config}
) )
@web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse)
@web_client.get("/config/content_type/{content_type}", response_class=HTMLResponse) def content_config_page(request: Request, content_type: str):
def content_config_page(request: Request, content_type: str):
if content_type not in VALID_CONTENT_TYPES: if content_type not in VALID_CONTENT_TYPES:
return templates.TemplateResponse("config.html", context={"request": request}) return templates.TemplateResponse("config.html", context={"request": request})
@@ -82,9 +87,8 @@ def content_config_page(request: Request, content_type: str):
}, },
) )
@web_client.get("/config/processor/conversation", response_class=HTMLResponse)
@web_client.get("/config/processor/conversation", response_class=HTMLResponse) def conversation_processor_config_page(request: Request):
def conversation_processor_config_page(request: Request):
default_copy = constants.default_config.copy() default_copy = constants.default_config.copy()
default_processor_config = default_copy["processor"]["conversation"] # type: ignore default_processor_config = default_copy["processor"]["conversation"] # type: ignore
default_processor_config = ConversationProcessorConfig( default_processor_config = ConversationProcessorConfig(
@@ -108,8 +112,3 @@ def conversation_processor_config_page(request: Request):
"current_config": current_processor_conversation_config, "current_config": current_processor_conversation_config,
}, },
) )
@web_client.get("/chat", response_class=FileResponse)
def chat_page():
return FileResponse(constants.web_directory / "chat.html")

1
src/khoj/utils/cli.py
View File

@@ -34,6 +34,7 @@ def cli(args=None):
help="Path to UNIX socket for server. Use to run server behind reverse proxy. Default: /tmp/uvicorn.sock", help="Path to UNIX socket for server. Use to run server behind reverse proxy. Default: /tmp/uvicorn.sock",
) )
parser.add_argument("--version", "-V", action="store_true", help="Print the installed Khoj version and exit") parser.add_argument("--version", "-V", action="store_true", help="Print the installed Khoj version and exit")
parser.add_argument("--demo", action="store_true", default=False, help="Run Khoj in demo mode")
args = parser.parse_args(args) args = parser.parse_args(args)

1
src/khoj/utils/state.py
View File

@@ -27,6 +27,7 @@ search_index_lock = threading.Lock()
SearchType = utils_config.SearchType SearchType = utils_config.SearchType
telemetry: List[Dict[str, str]] = [] telemetry: List[Dict[str, str]] = []
previous_query: str = None previous_query: str = None
demo: bool = False
if torch.cuda.is_available(): if torch.cuda.is_available():
# Use CUDA GPU # Use CUDA GPU