Moved data directly to the Space

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text

Data/{embeddings.npy → articles.hf/dataset.arrow}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b03978d1bf25675f47526cc5480bbe019a20f8c85bed35e35092a53d906fbeeb
-size 1713805184
+oid sha256:d9908602ac126bc96718c70b7d7d126a951526275c98ac98910d8852ca314431
+size 2294981896

Data/articles.hf/dataset_info.json

@@ -0,0 +1,35 @@
+{
+  "builder_name": null,
+  "citation": "",
+  "config_name": null,
+  "dataset_size": null,
+  "description": "",
+  "download_checksums": null,
+  "download_size": null,
+  "features": {
+    "id": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    },
+    "title": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    },
+    "abstract": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "post_processed": null,
+  "post_processing_size": null,
+  "size_in_bytes": null,
+  "splits": null,
+  "supervised_keys": null,
+  "task_templates": null,
+  "version": null
+}

Data/articles.hf/state.json

@@ -0,0 +1,14 @@
+{
+  "_data_files": [
+    {
+      "filename": "dataset.arrow"
+    }
+  ],
+  "_fingerprint": "5b2841faf59fb931",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_indexes": {},
+  "_output_all_columns": false,
+  "_split": null
+}

Data/articles.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5039ab8b65b9726fb81b3aabbb89774022c458e0c12a7c120a6fffe8c6775885
+size 3438919725

core.py

@@ -40,18 +40,10 @@ def load_model():
 @st.cache_resource
 def load_index():
     logging.info("Trying to load index")
-    index = datasets.load_dataset(
-        "eremeev-d/arxiv-abstracts",
-        use_auth_token=True,
-        split="train"
-    )
-    logging.info("Index succesfully loaded")
-    logging.info("Loading embeddings")
-    embeddings = np.load("Data/embeddings.npy")
-    logging.info("Loaded embeddings")
-    logging.info("Building index")
-    index.add_faiss_index_from_external_arrays(embeddings, 'embedding')
-    logging.info("Index built successfully")
+    index = datasets.Dataset.load_from_disk("Data/articles.hf")
+    logging.info("Articles dataset loaded")
+    index.load_faiss_index("embedding", "Data/articles.index")
+    logging.info("FAISS index loaded")
     return index