Merge pull request #43 from TheDataGuild/data/import-dennis-12-neural…

…-pdfs

Data/import dennis 12 neural pdfs

mind_palace/app.py

@@ -6,11 +6,11 @@
 from llama_index.query_engine import CitationQueryEngine
 
 openai.api_key = st.secrets.openai_key
-xml_dir = "./resources/xmls/12-pdfs-from-steve-aug-22/"
+xml_dir = "./resources/xmls/dennis-oct-10/"
 gpt_model = "gpt-3.5-turbo"
 
-st.set_page_config(page_title="Q&A with Steve's PDFs")
-st.title("Q&A with Steve's PDFs 💬")
+st.set_page_config(page_title="Q&A with Dennis's PDFs")
+st.title("Q&A with Dennis's PDFs 💬")
 
 with st.sidebar:
     st.markdown("Conversation History")

mind_palace/extract.py

@@ -7,7 +7,9 @@
 def _gen_document_dict(file_path) -> dict[str, TextNode]:
     xml = docs.load_tei_xml(file_path)
     doi = xml.header.doi
-    assert doi is not None
+    if doi is None:
+        print(f"DOI is None for {file_path}. Replacing with title instead.")
+        doi = xml.header.title
 
     try:
         title_node = docs.title(xml, doi)

resources/xmls/README.md

@@ -0,0 +1,5 @@
+These XMLs are generated from parsing the original PDFs with Grobid. Follow the official installation instructions [to build](https://grobid.readthedocs.io/en/latest/Install-Grobid/) and [run Grobid in command line batch mode](https://grobid.readthedocs.io/en/latest/Grobid-batch/) using this command:
+
+```sh
+$ java -Xmx4G -Djava.library.path=grobid-home/lib/lin-64:grobid-home/lib/lin-64/jep -jar grobid-core/build/libs/grobid-core-0.7.3-onejar.jar -gH grobid-home -dIn ~/Downloads/neural_decoding_papers -dOut ../mind-palace/resources/xmls/ -exe processFullText -ignoreAssets
+```