diff --git a/workshops/2024_Digital_History_Halle/01_SRU_Daten_beziehen.ipynb b/workshops/2024_Digital_History_Halle/01_SRU_Daten_beziehen.ipynb new file mode 100644 index 0000000..8a692ef --- /dev/null +++ b/workshops/2024_Digital_History_Halle/01_SRU_Daten_beziehen.ipynb @@ -0,0 +1,461 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "47549bfd-d4a1-4c0a-b044-cb526c1fa50b", + "metadata": {}, + "source": [ + "# Datenbezug über SRU und Umwandlung der Inhalte in eine Tabelle:" + ] + }, + { + "cell_type": "markdown", + "id": "9abe312c-b515-4330-b7cf-881a104ef632", + "metadata": {}, + "source": [ + "## Grundlagen zur SRU-Schnittstelle der DNB:\n", + "\n", + "Die DNB bietet Ihre Daten auf drei verschiedene \"Kataloge\" gesplittet an, von denen immer einer für eine Abfrage ausgewählt werden muss. Dies geschieht über eine Erweiterung der Base-URL. Zur Verfügung stehen folgende Kataloge:\n", + "\n", + "- Katalog der Deutschen Nationalbibliothek (DNB) - hierin befinden sich die Titeldaten\n", + "- Katalog des Deutschen Musikarchivs (DMA) - Datensätze des Deutschen Musikarchivs\n", + "- Katalog der Gemeinsamen Normdatei (GND) - hierin befinden sich die Normdaten\n", + "\n", + "Die erweiterungen für die URL sind folgende:\n", + "\n", + "- DNB: https://services.dnb.de/sru/dnb\n", + "- DMA: https://services.dnb.de/sru/dnb.dma\n", + "- GND: https://services.dnb.de/sru/authorities\n", + "\n", + "Werden die jeweiligen Bereiche ohne weitere Spezifikationen abgefragt, senden sie eine \"Explain-Response\" in XML zurück. \n", + "Mit Hilfe der Bibliothek BeautifulSoup kann die Antwort direkt in XML umgewandelt werden.\n", + "\n", + "[Weitere Informationen und Dokumentation zur SRU-Schnittstelle](https://www.dnb.de/sru)\n", + "\n", + "### Laden benötigter Zusatzbibliotheken: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51bb5a4b-e64e-41e7-ad2a-b96b1de4a11e", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from bs4 import BeautifulSoup as soup\n", + "import unicodedata\n", + "from lxml import etree\n", + "import pandas as pd\n" + ] + }, + { + "cell_type": "markdown", + "id": "87404a94-7132-4f99-8279-2a25e179a0e0", + "metadata": {}, + "source": [ + "## SRU-Anfrage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ee29b90-36ed-4a73-b52f-9bc54d04d5ff", + "metadata": {}, + "outputs": [], + "source": [ + "#Base-URL für den Katalog der DNB (Titeldaten):\n", + "dnb = requests.get(\"https://services.dnb.de/sru/dnb\")\n", + "\n", + "#Einfache Amfrage an die Schnittstelle ohne Suche \n", + "response = soup(dnb.content, features=\"xml\") #Parsen der Schnittstellenantwort als xml\n", + "print(response.prettify()[0:500]) #Einschränken der Ausgabe auf die ersten 500 Zeichen" + ] + }, + { + "cell_type": "markdown", + "id": "ffc1a87a-e58f-4fca-9931-f6cf4ef2e015", + "metadata": {}, + "source": [ + "Für eine Suchanfrage an die Daten der DNB wird nun zunächst über die Wahl der URL der Katalog definiert. Mit Hilfe der Variable parameter werdem dann alle weiteren Parameter, die die SRU-Schnittstelle benötigt, übergeben." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c01637fc-a150-4895-81ed-c9eca69bff89", + "metadata": {}, + "outputs": [], + "source": [ + "#Base-URL:\n", + "dnb_url = \"https://services.dnb.de/sru/dnb\"\n", + "\n", + "#Parameter, die mit der Anfrage übergeben werden sollen: \n", + "parameter = {'version' : '1.1' , \n", + " 'operation' : 'searchRetrieve' , \n", + " 'query' : 'Klimawandel and jhr=2010', \n", + " 'recordSchema' : 'MARC21-xml'} \n", + "\n", + "#Abfrage der Schnittstelle:\n", + "r = requests.get(dnb_url, params = parameter)\n", + "#Parsen der Antwort in xml:\n", + "response = soup(r.content, features=\"xml\")\n", + "#Überführen der einzelnen, in der Antwort enthaltenen Datensätze/Treffer ('Records') in eine Liste:\n", + "result = response.find_all('record', {'type':'Bibliographic'})\n", + "print(response.prettify()[0:750])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e013ff23-cc54-4c41-8d17-d60c872239c1", + "metadata": {}, + "outputs": [], + "source": [ + "number = response.find('numberOfRecords')\n", + "print(number.text, 'Ergebnisse')\n", + "print(len(result))" + ] + }, + { + "cell_type": "markdown", + "id": "4e071f81-62bb-4c6e-bf4b-d14cfe11eb7b", + "metadata": {}, + "source": [ + "Wichtig: Die SRU-Schnittstelle liefert per default zunächst immer nur 10 Datensätze auf einmal aus, maximal 100. " + ] + }, + { + "cell_type": "markdown", + "id": "45fd84cb-8cef-4108-a54b-75cb0b3a9fa1", + "metadata": {}, + "source": [ + "### Umwandlung in eine Funktion:" + ] + }, + { + "cell_type": "markdown", + "id": "ad50c950-7c48-4d0b-a262-6bac2f1f7f88", + "metadata": {}, + "source": [ + "Funktion, die dank while-Schleife alle Datensätze der Anfrage sammelt: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c09f414b-76e6-4715-9180-9d4d0a11fda3", + "metadata": {}, + "outputs": [], + "source": [ + "def dnb_sru(query):\n", + " \n", + " base_url = \"https://services.dnb.de/sru/dnb\"\n", + " params = {'recordSchema' : 'MARC21-xml',\n", + " 'operation': 'searchRetrieve',\n", + " 'version': '1.1',\n", + " 'maximumRecords': '100',\n", + " 'query': query\n", + " }\n", + " \n", + " i = 0 \n", + " records = []\n", + "\n", + " while i % 100 == 0: \n", + " params.update({'startRecord': i}) \n", + " r = requests.get(base_url, params=params)\n", + " xml = soup(r.content, features=\"xml\")\n", + " result = xml.find_all('record', {'type':'Bibliographic'})\n", + " records+=result\n", + " num_results = len(result)\n", + " if num_results != 100:\n", + " break\n", + " else: \n", + " i += 100 \n", + " \n", + " return records" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f7f4b07-e684-4e0a-84dc-abc7cb0ad1e1", + "metadata": {}, + "outputs": [], + "source": [ + "records = dnb_sru('Studentenbewegung') #and location=onlinefree and cod=pdf')\n", + "print(len(records), 'Ergebnisse')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7446fb3e-eaf3-494f-a571-8b6e301b6b55", + "metadata": {}, + "outputs": [], + "source": [ + "print(records[137].prettify())" + ] + }, + { + "cell_type": "markdown", + "id": "72b8f418-4d6d-494f-bbf0-d1b402134aab", + "metadata": {}, + "source": [ + "## Inhalte aus dem XML extrahieren: " + ] + }, + { + "cell_type": "markdown", + "id": "b7b92abd-1f48-4c57-8068-40ee6ec0e311", + "metadata": {}, + "source": [ + "### Einblick: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a28cefc-851d-42a2-9806-0aa7e7ec3cb3", + "metadata": {}, + "outputs": [], + "source": [ + "test_record = records[137]\n", + "test_record = str(test_record)\n", + "print(test_record)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad21a7fc-f387-440f-bcc2-db434b0106bf", + "metadata": {}, + "outputs": [], + "source": [ + "# Parsen des XML: \n", + "root = etree.fromstring(test_record)\n", + "print(root)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b6fa735-5862-49bb-9a98-027c51860399", + "metadata": {}, + "outputs": [], + "source": [ + "# Extrahieren des Inhalts des Controlfields \"001\"\n", + "controlfield_001 = root.find(\".//controlfield[@tag='001']\")\n", + "print(controlfield_001)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a308b9a-2076-43f6-bef4-d8b7e74b4e7d", + "metadata": {}, + "outputs": [], + "source": [ + "# Definieren des Namespaces: \n", + "namespace = {'marc': 'http://www.loc.gov/MARC21/slim'}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7bf43c9-7438-492b-b261-3ea969f518f0", + "metadata": {}, + "outputs": [], + "source": [ + "controlfield_001 = root.find('.//marc:controlfield[@tag=\"001\"]', namespaces=namespace)#.text\n", + "print(controlfield_001)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6efa98d3-fd1c-423a-9b4c-6e296508a7d8", + "metadata": {}, + "outputs": [], + "source": [ + "datafield_245_a = root.xpath('.//marc:datafield[@tag=\"245\"]/marc:subfield[@code=\"a\"]/text()', namespaces=namespace)\n", + "print(datafield_245_a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87c4240f-c059-4d4c-b5e3-3790747cd3ad", + "metadata": {}, + "outputs": [], + "source": [ + "print(type(datafield_245_a))" + ] + }, + { + "cell_type": "markdown", + "id": "49c3a719-2422-44bd-8dac-59b398242cbe", + "metadata": {}, + "source": [ + "#### Erstellen einer einfachen Funktion: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6312438c-db13-4412-ab4d-6263761e0d00", + "metadata": {}, + "outputs": [], + "source": [ + "def simple_parse_record(record):\n", + "\n", + " record = str(record).replace('\\x98', '').replace('\\x9c', '')\n", + " ns = {\"marc\":\"http://www.loc.gov/MARC21/slim\"}\n", + " xml = etree.fromstring(unicodedata.normalize(\"NFC\", str(record)))\n", + " \n", + " #idn\n", + " idn = xml.xpath(\"marc:controlfield[@tag = '001']\", namespaces=ns)\n", + " try:\n", + " idn = idn[0].text\n", + " except:\n", + " idn = 'fail'\n", + " \n", + " # titel\n", + " titel = xml.xpath(\"marc:datafield[@tag = '245']/marc:subfield[@code = 'a']\", namespaces=ns)\n", + " try:\n", + " titel = titel[0].text\n", + " except:\n", + " titel = \"unkown\"\n", + " \n", + " meta_dict = {\"idn\":idn, \"titel\":titel}\n", + " #print(meta_dict)\n", + " \n", + " return meta_dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "177ad51e-5f79-40d5-a9f4-9085f44d2264", + "metadata": {}, + "outputs": [], + "source": [ + "output = [simple_parse_record(record) for record in records]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0efcd4e9-4d37-40ca-a154-501ca81e56b9", + "metadata": {}, + "outputs": [], + "source": [ + "df_simple = pd.DataFrame(output)\n", + "df_simple" + ] + }, + { + "cell_type": "markdown", + "id": "f2a1cd93-0481-44a8-8e5d-4ee173237963", + "metadata": {}, + "source": [ + "### Schönere, besser zu bedienende Funktion: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1c8e1f1-c9c5-4eab-b070-6f85e53365b1", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_record(record):\n", + " \n", + " record = str(record).replace('\\x98', '').replace('\\x9c', '')\n", + " ns = {\"marc\": \"http://www.loc.gov/MARC21/slim\"}\n", + " xml = etree.fromstring(unicodedata.normalize(\"NFC\", str(record)))\n", + " \n", + " \n", + " def extract_text(xpath_query):\n", + " fields = xml.xpath(xpath_query, namespaces=ns)\n", + " if fields:\n", + " return \"; \".join(field.text for field in fields if field.text)\n", + " return \"unknown\"\n", + "\n", + " \n", + " idn = extract_text(\"marc:controlfield[@tag='001']\")\n", + " titel = extract_text(\"marc:datafield[@tag='245']/marc:subfield[@code='a']\")\n", + " author = extract_text(\"marc:datafield[@tag='100']/marc:subfield[@code='a']\")\n", + " author_rela = extract_text(\"marc:datafield[@tag='100']/marc:subfield[@code='e']\")\n", + " author_gnd = extract_text(\"marc:datafield[@tag='100']/marc:subfield[@code='0']\")\n", + " links = extract_text(\"marc:datafield[@tag='856']/marc:subfield[@code='u']\")\n", + " \n", + " #add_author = extract_text(\"marc:datafield[@tag='700']/marc:subfield[@code='a']\")\n", + " #add_author_rela = extract_text(\"marc:datafield[@tag='700']/marc:subfield[@code='e']\")\n", + "\n", + " \n", + " return {\"idn\": idn, \"titel\": titel, \"author\": author, \"rela\":author_rela, \"gnd\":author_gnd} #, \"links\":links}\n", + " #\"additional author\":add_author, \"added_rela\": add_author_rela}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a7c5e05-1648-4fa9-88ae-fabec82c5ec8", + "metadata": {}, + "outputs": [], + "source": [ + "result = [parse_record(record) for record in records]\n", + "df = pd.DataFrame(result)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "8a4035fd-5e5b-4cfb-b514-c694aac1f6c2", + "metadata": {}, + "source": [ + "## Speichern: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bcc6a9a9-1b6b-43fd-af2e-1665082eb2cc", + "metadata": {}, + "outputs": [], + "source": [ + "df.to_csv(\"Studentenbewegung.csv\", index=False, encoding=\"utf-8\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e000f66-be3d-47e2-92f9-68ee8737fd7f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}