chip_prep_background.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['OH_Addison_224687_1960_24000_geo.tif', 'OH_Addison_224689_1960_24000_geo.tif', 'OH_Addison_226205_1960_24000_geo.tif', 'OH_Albany_224714_1960_24000_geo.tif', 'OH_Albany_224716_1960_24000_geo.tif', 'OH_Dalzell_224610_1961_24000_geo.tif', 'OH_Dalzell_224612_1961_24000_geo.tif', 'OH_Lowell_225659_1961_24000_geo.tif', 'OH_Lowell_225661_1961_24000_geo.tif', 'OH_Macksburg_225682_1961_24000_geo.tif', 'OH_Macksburg_225684_1961_24000_geo.tif', 'OH_Rutland_227264_1960_24000_geo.tif', 'OH_Rutland_227266_1960_24000_geo.tif', 'OH_Sarahsville_226922_1961_24000_geo.tif', 'OH_Sarahsville_226923_1961_24000_geo.tif', 'OH_Shade_226959_1960_24000_geo.tif', 'OH_Shade_226961_1960_24000_geo.tif', 'OH_Stafford_227035_1961_24000_geo.tif', 'OH_Stafford_227037_1961_24000_geo.tif', 'OH_Stafford_228181_2002_24000_geo.tif', 'OH_Summerfield_226730_1961_24000_geo.tif', 'OH_Summerfield_226731_1961_24000_geo.tif', 'OH_Summerfield_228206_2002_24000_geo.tif']\n"
     ]
    }
   ],
   "source": [
    "#Read in libraries/extensions\n",
    "import arcpy\n",
    "arcpy.CheckOutExtension(\"ImageAnalyst\")\n",
    "from arcpy.ia import *\n",
    "from arcpy import env\n",
    "from arcpy.sa import *\n",
    "import os\n",
    "import os.path\n",
    "\n",
    "#Define input folders\n",
    "in_fold = \"E:/topo_proj/topo_check/\"\n",
    "in_toposF = \"oh_needed_topos2/\"\n",
    "in_minesF = \"oh_topo_mines2/\"\n",
    "in_quadsF = \"oh_topo_quads/\"\n",
    "chips_outF = in_fold + \"processing/oh_chips_background/\"\n",
    "to_8bitF = in_fold + \"processing/oh_topo8bit_background/\"\n",
    "\n",
    "#Create new directories\n",
    "os.mkdir(chips_outF)\n",
    "os.mkdir(to_8bitF)\n",
    "\n",
    "#List all topo maps\n",
    "arcpy.env.workspace =  in_fold + in_toposF\n",
    "chips = arcpy.ListRasters()\n",
    "print(chips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['OH_Addison_224687_1960_24000_geo', 'OH_Addison_224689_1960_24000_geo', 'OH_Addison_226205_1960_24000_geo', 'OH_Albany_224714_1960_24000_geo', 'OH_Albany_224716_1960_24000_geo', 'OH_Dalzell_224610_1961_24000_geo', 'OH_Dalzell_224612_1961_24000_geo', 'OH_Lowell_225659_1961_24000_geo', 'OH_Lowell_225661_1961_24000_geo', 'OH_Macksburg_225682_1961_24000_geo', 'OH_Macksburg_225684_1961_24000_geo', 'OH_Rutland_227264_1960_24000_geo', 'OH_Rutland_227266_1960_24000_geo', 'OH_Sarahsville_226922_1961_24000_geo', 'OH_Sarahsville_226923_1961_24000_geo', 'OH_Shade_226959_1960_24000_geo', 'OH_Shade_226961_1960_24000_geo', 'OH_Stafford_227035_1961_24000_geo', 'OH_Stafford_227037_1961_24000_geo', 'OH_Stafford_228181_2002_24000_geo', 'OH_Summerfield_226730_1961_24000_geo', 'OH_Summerfield_226731_1961_24000_geo', 'OH_Summerfield_228206_2002_24000_geo']\n"
     ]
    }
   ],
   "source": [
    "#Make list and remove file extensions\n",
    "chip_n = list()\n",
    "for c in chips:\n",
    "    c1 = os.path.basename(c)\n",
    "    c2 = os.path.splitext(c1)[0]\n",
    "    chip_n.append(c2)\n",
    "print(chip_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make chips for each topo map\n",
    "for cr in chip_n:\n",
    "    #Set workspace\n",
    "    arcpy.env.workspace = in_fold + in_toposF\n",
    "    #Set local variables and make folders\n",
    "    quadNm = cr.split(\"_\")[1].replace(\" \", \"_\")\n",
    "    os.mkdir(chips_outF + cr)\n",
    "    subdir = chips_outF + cr + \"/\"\n",
    "    out_folder=subdir\n",
    "    #Read in topo\n",
    "    inRaster = in_fold + in_toposF + cr  + \".tif\"\n",
    "    #Copy topo to 8-bit PNG\n",
    "    inRaster2 = arcpy.CopyRaster_management(inRaster, to_8bitF + cr + \".png\",\n",
    "                                \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                  \"NONE\")\n",
    "    #Read in Mines\n",
    "    in_mines = in_fold + in_minesF + cr  + \".shp\"\n",
    "    in_training =  in_fold + in_minesF + cr  + \".shp\"\n",
    "    #Add and populate class value field if it doesn't already exist\n",
    "    if 'classvalue' not in [f.name for f in arcpy.ListFields(in_training)]:\n",
    "        arcpy.AddField_management(in_training, \"classvalue\", \"SHORT\")\n",
    "        arcpy.CalculateField_management(in_training, \"classvalue\", 1, \"PYTHON3\")\n",
    "    #Define image chip parameters\n",
    "    image_chip_format = \"PNG\"\n",
    "    tile_size_x = \"128\"\n",
    "    tile_size_y = \"128\"\n",
    "    stride_x= \"128\"\n",
    "    stride_y= \"128\"\n",
    "    output_nofeature_tiles= \"ALL_TILES\"\n",
    "    metadata_format= \"Classified_Tiles\"\n",
    "    start_index = 0\n",
    "    classvalue_field = \"classvalue\"\n",
    "    buffer_radius = 0\n",
    "    in_mask_polygons = in_fold + in_quadsF + cr + \".shp\"\n",
    "    rotation_angle = 0\n",
    "    reference_system = \"MAP_SPACE\"\n",
    "    processing_mode = \"PROCESS_AS_MOSAICKED_IMAGE\"\n",
    "    blacken_around_feature = \"NO_BLACKEN\"\n",
    "    crop_mode = \"FIXED_SIZE\"\n",
    "\n",
    "    # Create image chips\n",
    "    ExportTrainingDataForDeepLearning(inRaster2, out_folder, in_training, \n",
    "                 image_chip_format,tile_size_x, tile_size_y, stride_x, \n",
    "                 stride_y,output_nofeature_tiles, metadata_format, start_index, \n",
    "                 classvalue_field, buffer_radius, in_mask_polygons, rotation_angle, \n",
    "                 reference_system, processing_mode, blacken_around_feature, crop_mode)\n",
    "    #Make new labels and image directory\n",
    "    os.mkdir(subdir + \"labels2\")\n",
    "    os.mkdir(subdir + \"images2\")\n",
    "    arcpy.env.workspace = subdir + \"images\"\n",
    "    imgchips = arcpy.ListRasters()\n",
    "    #Copy all only background chips and make 0 masks. \n",
    "    for ic in imgchips:\n",
    "        if os.path.isfile(subdir + \"labels/\" + ic):\n",
    "            pass\n",
    "        else:\n",
    "            makeZeros = IsNull(subdir + \"images/\" + ic)\n",
    "            arcpy.CopyRaster_management(subdir + \"images/\" + ic, subdir + \"images2/\" + ic,\n",
    "                                    \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                    \"NONE\")\n",
    "            arcpy.CopyRaster_management(makeZeros, subdir + \"labels2/\" + ic,\n",
    "                                    \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                    \"NONE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['VA_Coeburn_184602_1957_24000_geo.tif', 'VA_Coeburn_184605_1957_24000_geo.tif', 'VA_Duty_184823_1958_24000_geo.tif', 'VA_Duty_184825_1958_24000_geo.tif', 'VA_Grundy_185240_1963_24000_geo.tif', 'VA_Grundy_185241_1963_24000_geo.tif', 'VA_Haysi_185326_1963_24000_geo.tif', 'VA_Haysi_185327_1963_24000_geo.tif', 'VA_Honaker_185407_1968_24000_geo.tif', 'VA_Honaker_185409_1968_24000_geo.tif', 'VA_Nora_186109_1958_24000_geo.tif', 'VA_Nora_8031190_1958_24000_geo.tif', 'VA_Norton_186163_1957_24000_geo.tif', 'VA_Norton_186165_1957_24000_geo.tif', 'VA_Pound_186331_1957_24000_geo.tif', 'VA_Pound_186333_1957_24000_geo.tif', 'VA_Prater_186345_1963_24000_geo.tif', 'VA_Prater_186347_1963_24000_geo.tif', 'VA_Richlands_186494_1968_24000_geo.tif', 'VA_Richlands_186496_1968_24000_geo.tif', 'VA_Vansant_187042_1963_24000_geo.tif', 'VA_Vansant_187044_1963_24000_geo.tif', 'VA_Wise_187272_1957_24000_geo.tif', 'VA_Wise_187273_1957_24000_geo.tif', 'VA_Wise_187274_1957_24000_geo.tif']\n"
     ]
    }
   ],
   "source": [
    "import arcpy\n",
    "arcpy.CheckOutExtension(\"ImageAnalyst\")\n",
    "from arcpy.ia import *\n",
    "from arcpy import env\n",
    "from arcpy.sa import *\n",
    "import os\n",
    "import os.path\n",
    "\n",
    "in_fold = \"E:/topo_proj/topo_check/\"\n",
    "in_toposF = \"va_needed_topos2/\"\n",
    "in_minesF = \"va_topo_mines2/\"\n",
    "in_quadsF = \"va_topo_quads/\"\n",
    "chips_outF = in_fold + \"processing/va_chips_background/\"\n",
    "to_8bitF = in_fold + \"processing/va_topo8bit_background/\"\n",
    "#os.mkdir(chips_outF)\n",
    "#os.mkdir(to_8bitF)\n",
    "\n",
    "arcpy.env.workspace =  in_fold + in_toposF\n",
    "chips = arcpy.ListRasters()\n",
    "print(chips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['VA_Coeburn_184602_1957_24000_geo', 'VA_Coeburn_184605_1957_24000_geo', 'VA_Duty_184823_1958_24000_geo', 'VA_Duty_184825_1958_24000_geo', 'VA_Grundy_185240_1963_24000_geo', 'VA_Grundy_185241_1963_24000_geo', 'VA_Haysi_185326_1963_24000_geo', 'VA_Haysi_185327_1963_24000_geo', 'VA_Honaker_185407_1968_24000_geo', 'VA_Honaker_185409_1968_24000_geo', 'VA_Nora_186109_1958_24000_geo', 'VA_Nora_8031190_1958_24000_geo', 'VA_Norton_186163_1957_24000_geo', 'VA_Norton_186165_1957_24000_geo', 'VA_Pound_186331_1957_24000_geo', 'VA_Pound_186333_1957_24000_geo', 'VA_Prater_186345_1963_24000_geo', 'VA_Prater_186347_1963_24000_geo', 'VA_Richlands_186494_1968_24000_geo', 'VA_Richlands_186496_1968_24000_geo', 'VA_Vansant_187042_1963_24000_geo', 'VA_Vansant_187044_1963_24000_geo', 'VA_Wise_187272_1957_24000_geo', 'VA_Wise_187273_1957_24000_geo', 'VA_Wise_187274_1957_24000_geo']\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "chip_n = list()\n",
    "for c in chips:\n",
    "    c1 = os.path.basename(c)\n",
    "    c2 = os.path.splitext(c1)[0]\n",
    "    chip_n.append(c2)\n",
    "print(chip_n)\n",
    "print(len(chip_n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['VA_Wise_187273_1957_24000_geo', 'VA_Wise_187274_1957_24000_geo']\n"
     ]
    }
   ],
   "source": [
    "chip_n2 = chip_n[23:]\n",
    "print(chip_n2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cr in chip_n2:\n",
    "    # Set local variables\n",
    "    arcpy.env.workspace = in_fold + in_toposF\n",
    "    quadNm = cr.split(\"_\")[1].replace(\" \", \"_\")\n",
    "    os.mkdir(chips_outF + cr)\n",
    "    subdir = chips_outF + cr + \"/\"\n",
    "    out_folder=subdir\n",
    "    inRaster = in_fold + in_toposF + cr  + \".tif\"\n",
    "    inRaster2 = arcpy.CopyRaster_management(inRaster, to_8bitF + cr + \".png\",\n",
    "                                \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                  \"NONE\")\n",
    "    in_mines = in_fold + in_minesF + cr  + \".shp\"\n",
    "    in_training =  in_fold + in_minesF + cr  + \".shp\"\n",
    "    if 'classvalue' not in [f.name for f in arcpy.ListFields(in_training)]:\n",
    "        arcpy.AddField_management(in_training, \"classvalue\", \"SHORT\")\n",
    "        arcpy.CalculateField_management(in_training, \"classvalue\", 1, \"PYTHON3\")\n",
    "    image_chip_format = \"PNG\"\n",
    "    tile_size_x = \"128\"\n",
    "    tile_size_y = \"128\"\n",
    "    stride_x= \"128\"\n",
    "    stride_y= \"128\"\n",
    "    output_nofeature_tiles= \"ALL_TILES\"\n",
    "    metadata_format= \"Classified_Tiles\"\n",
    "    start_index = 0\n",
    "    classvalue_field = \"classvalue\"\n",
    "    buffer_radius = 0\n",
    "    in_mask_polygons = in_fold + in_quadsF + cr + \".shp\"\n",
    "    rotation_angle = 0\n",
    "    reference_system = \"MAP_SPACE\"\n",
    "    processing_mode = \"PROCESS_AS_MOSAICKED_IMAGE\"\n",
    "    blacken_around_feature = \"NO_BLACKEN\"\n",
    "    crop_mode = \"FIXED_SIZE\"\n",
    "\n",
    "    # Execute \n",
    "    ExportTrainingDataForDeepLearning(inRaster2, out_folder, in_training, \n",
    "                 image_chip_format,tile_size_x, tile_size_y, stride_x, \n",
    "                 stride_y,output_nofeature_tiles, metadata_format, start_index, \n",
    "                 classvalue_field, buffer_radius, in_mask_polygons, rotation_angle, \n",
    "                 reference_system, processing_mode, blacken_around_feature, crop_mode)\n",
    "    os.mkdir(subdir + \"labels2\")\n",
    "    os.mkdir(subdir + \"images2\")\n",
    "    arcpy.env.workspace = subdir + \"images\"\n",
    "    imgchips = arcpy.ListRasters()\n",
    "    for ic in imgchips:\n",
    "        if os.path.isfile(subdir + \"labels/\" + ic):\n",
    "            pass\n",
    "        else:\n",
    "            makeZeros = IsNull(subdir + \"images/\" + ic)\n",
    "            arcpy.CopyRaster_management(subdir + \"images/\" + ic, subdir + \"images2/\" + ic,\n",
    "                                    \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                    \"NONE\")\n",
    "            arcpy.CopyRaster_management(makeZeros, subdir + \"labels2/\" + ic,\n",
    "                                    \"\", \"\",256,\"NONE\",\"NONE\",\"8_BIT_UNSIGNED\",\"NONE\",\"NONE\", \"PNG\", \n",
    "                                    \"NONE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}