Add support for rate limiting the amount of data indexed

- Add a dependency on the indexer API endpoint that rounds up the amount of data indexed and uses that to determine whether the next set of data should be processed
- Delete any files that are being removed for adminstering the calculation
- Show current amount of data indexed in the config page

tests/test_client.py

@@ -125,6 +125,34 @@ def test_regenerate_with_invalid_content_type(client):
     assert response.status_code == 422
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.django_db(transaction=True)
+def test_index_update_big_files(client):
+    state.billing_enabled = True
+    # Arrange
+    files = get_big_size_sample_files_data()
+    headers = {"Authorization": "Bearer kk-secret"}
+
+    # Act
+    response = client.post("/api/v1/index/update", files=files, headers=headers)
+
+    # Assert
+    assert response.status_code == 429
+
+
+@pytest.mark.django_db(transaction=True)
+def test_index_update_big_files_no_billing(client):
+    # Arrange
+    files = get_big_size_sample_files_data()
+    headers = {"Authorization": "Bearer kk-secret"}
+
+    # Act
+    response = client.post("/api/v1/index/update", files=files, headers=headers)
+
+    # Assert
+    assert response.status_code == 200
+
+
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.django_db(transaction=True)
 def test_index_update(client):
@@ -421,3 +449,13 @@ def get_sample_files_data():
         ),
         ("files", ("path/to/filename2.md", "**Understanding science through the lens of art**", "text/markdown")),
     ]
+
+
+def get_big_size_sample_files_data():
+    big_text = "a" * (25 * 1024 * 1024)  # a string of approximately 25 MB
+    return [
+        (
+            "files",
+            ("path/to/filename.org", big_text, "text/org"),
+        )
+    ]