pd_io_json_route.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pd_io_json_route.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyO6+vsRsE8HdWJcQ5dAwDh5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jugalpanchal/py-pilgrim/blob/main/pd_io_json_route.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YvO_CunNWuZt"
      },
      "source": [
        "import os\n",
        "import json\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePm1rl7wW3wn"
      },
      "source": [
        "def get_record(file_path, keys_to_use):\n",
        "  \"\"\"Process single json file and return a tuple containing specific fields\"\"\"\n",
        "\n",
        "  with open(file_path) as artwork_file:\n",
        "    content = json.load(artwork_file)\n",
        "    #print(content)\n",
        "\n",
        "    records = []\n",
        "    for field in keys_to_use:\n",
        "      records.append(content[field])\n",
        "\n",
        "    return tuple(records)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "03u_6JevXfbU",
        "outputId": "7430916c-6159-4fdc-b5e0-a480d4426042"
      },
      "source": [
        "KEYS_TO_USE = ['id', 'all_artists', 'title', 'medium', 'acquisitionYear', 'height', 'width']\n",
        "json_file_path = os.path.join('collection-master/artworks/a/000', 'a00001-1035.json')\n",
        "\n",
        "record_tuple = get_record(json_file_path, KEYS_TO_USE)\n",
        "print(record_tuple)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1035, 'Robert Blake', 'A Figure Bowing before a Seated Old Man with his Arm Outstretched in Benediction. Verso: Indecipherable Sketch', 'Watercolour, ink, chalk and graphite on paper. Verso: graphite on paper', 1922, '419', '394')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EFBQxUjAZPb2"
      },
      "source": [
        "def read_records(json_root_path, keys_to_use):\n",
        "  \"\"\" Traverse each json file in the directory\"\"\"\n",
        "  artworks = []\n",
        "  for root, _, files in os.walk(json_root_path):\n",
        "    for f in files:\n",
        "      if f.endswith('json'):\n",
        "        file_path = os.path.join(root, f)\n",
        "        print(file_path)\n",
        "        record = get_record(file_path, keys_to_use)\n",
        "        artworks.append(record)\n",
        "      break # read first file only in each diretory\n",
        "\n",
        "  df = pd.DataFrame.from_records(artworks, columns=keys_to_use, index=\"id\")\n",
        "  return df"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GPZvAp4xdHCa",
        "outputId": "f9210edb-a6d7-4d76-fb92-ce31da70eb20"
      },
      "source": [
        "KEYS_TO_USE = ['id', 'all_artists', 'title', 'medium', 'acquisitionYear', 'height', 'width']\n",
        "json_root_path = os.path.join('collection-master', 'artworks')\n",
        "\n",
        "df = read_records(json_root_path, KEYS_TO_USE)\n",
        "print(df)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "collection-master/artworks/a/000/a00001-1035.json\n",
            "       all_artists  ... width\n",
            "id                  ...      \n",
            "1035  Robert Blake  ...   394\n",
            "\n",
            "[1 rows x 6 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RYSgGf0nR0wR"
      },
      "source": [
        "# export dataframe in a json file\n",
        "df.to_json('artworks_1.json') # only data\n",
        "df.to_json('artworks_2.json', orient = 'table') # schema and data"
      ],
      "execution_count": 6,
      "outputs": []
    }
  ]
}