Make_TB_file.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make the Darwin Core Archive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook takes the json results from the \"get_effechecka_data\" notebook and the dictionaries generated in the \"build_taxon\" to make the files needed for a Darwin Core Archive. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code below imports the necessary libraries and data files and creates the necessary out_files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import re\n",
    "\n",
    "country = 'Guatemala' #user sets country - needs proper capitalization\n",
    "country_ = re.sub('\\s', '_', country)\n",
    "country_ = country_.lower()\n",
    "\n",
    "in_file = open('/Users/annethessen/effechecka_country_results/guatemala/' + country_ + '.tsv', 'r') #this is the results from effechecka\n",
    "#in_file = open('sweden.tsv', 'r')\n",
    "out_file = open('tb_measurement.txt', 'w')\n",
    "out_file_t = open('tb_taxon.txt', 'w')\n",
    "out_file_c = open('tb_occurrence.txt', 'w')\n",
    "f = open('parent_dict.p', 'rb')\n",
    "g = open('taxon_id.p', 'rb')\n",
    "h = open('id_dict.p', 'rb')\n",
    "i = open('polygon_dict.p', 'rb')\n",
    "\n",
    "parents = pickle.load(f) #unpickles the dictionaries\n",
    "ids = pickle.load(g)\n",
    "country_ids = pickle.load(h)\n",
    "polygons = pickle.load(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cell below sets the country id, and the polygon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "country_uri = country_ids[country]\n",
    "polygon = polygons[country_uri]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cell below contains functions used in the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#this function normalizes the length of the pipe-delimited higher classification so the rest of the code will work.\n",
    "#I noticed in the Australia and Argentina lists, instead of having blanks, the higher classifications were different \n",
    "#lengths.\n",
    "def norm_len(hi_class):\n",
    "    #print(len(hi_class))\n",
    "    skip = False\n",
    "    if len(hi_class) == 1:\n",
    "        skip = True\n",
    "    elif len(hi_class) < 3:\n",
    "        hi_class.insert(3, '')\n",
    "        hi_class.insert(4, '')\n",
    "        hi_class.insert(5, '')\n",
    "        hi_class.insert(6, '')\n",
    "        hi_class.insert(7, '')\n",
    "    elif len(hi_class) < 4:\n",
    "        hi_class.insert(4, '')\n",
    "        hi_class.insert(5, '')\n",
    "        hi_class.insert(6, '')\n",
    "        hi_class.insert(7, '')        \n",
    "    elif len(hi_class) < 6:\n",
    "        hi_class.insert(5, '')\n",
    "        hi_class.insert(6, '')\n",
    "        hi_class.insert(7, '')\n",
    "    elif len(hi_class) < 7:\n",
    "        hi_class.insert(3, '')\n",
    "        name = hi_class[5].split(' ')\n",
    "        if len(name) < 2:\n",
    "            skip = True\n",
    "        else:\n",
    "            print(hi_class)\n",
    "            print(name)\n",
    "            genus = name[0]\n",
    "            spec = name[1]\n",
    "            hi_class.insert(5, genus)\n",
    "            hi_class.insert(6, spec)\n",
    "    elif len(hi_class) < 8:\n",
    "        name = hi_class[6].split(' ')\n",
    "        if len(name) < 2:\n",
    "            skip = True\n",
    "        else:\n",
    "            spec = name[1]\n",
    "            hi_class.insert(6, spec)\n",
    "    else:\n",
    "        pass\n",
    "    return hi_class, skip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code below takes a json file from effechecka for a single country and iterates over the list of taxa to make the data files for TraitBank."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total taxa 13129\n",
      "total records 499923\n",
      "complete\n"
     ]
    }
   ],
   "source": [
    "kingdoms = []\n",
    "phyla = []\n",
    "classes = []\n",
    "orders = []\n",
    "families = []\n",
    "taxa = []\n",
    "genera = []\n",
    "counter = 100000\n",
    "\n",
    "record_total = 0\n",
    "in_file.seek(0)\n",
    "next(in_file)\n",
    "for line in in_file:\n",
    "    line = line.strip('\\n')\n",
    "    row = line.split('\\t')\n",
    "    taxon_string = row[1].split('|') #the json result includes a pipe-delimited higher classification\n",
    "    for i,j in enumerate(taxon_string): #this for loop removes any incertae sedis. We don't want that as a taxon.\n",
    "        if j == 'incertae sedis':\n",
    "            taxon_string[i] = ''\n",
    "    taxon_string, r = norm_len(taxon_string) #using the function to normalize lengths\n",
    "    if r == True:\n",
    "        continue\n",
    "    #print(taxon_string)\n",
    "    kingdom = taxon_string[0].title()\n",
    "    phylum = taxon_string[1].title()\n",
    "    class_ = taxon_string[2].title()\n",
    "    order = taxon_string[3].title()\n",
    "    family = taxon_string[4].title()\n",
    "    genus = taxon_string[5].title()\n",
    "    species = taxon_string[6]\n",
    "    if species == '' or genus == '': #if its not identified to species, then we don't want it\n",
    "        continue\n",
    "    else:\n",
    "        taxon = genus + ' ' + species\n",
    "        if taxon in taxa: #remove accidental duplication\n",
    "            #print(taxon)\n",
    "            continue\n",
    "        else:\n",
    "            taxa.append(taxon)\n",
    "            n = row[2] #this is our sample size\n",
    "            record_total = record_total + int(n)\n",
    "            meas_id = 'M' + str(counter)\n",
    "            t_id = ids[taxon] #looking up taxon identifier\n",
    "            occur_id = 'C' + t_id\n",
    "            par_id = parents[taxon] #looking up the parent identifier\n",
    "            #the below lines of output are writing to the darwin core files\n",
    "            out_file.write(meas_id + '\\t' + occur_id + '\\t' + '' + '\\t' + 'true' + '\\t' + 'http://eol.org/schema/terms/Present' + '\\t' + 'http://www.geonames.org/' + str(country_uri) + '\\t' + 'R01|R02' + '\\t' + 'Compiler: Anne E Thessen' + '\\t' + 'http://gimmefreshdata.github.io/?limit=5000000&taxonSelector=' + taxon + '&traitSelector=&wktString=' + polygon + '\\n')\n",
    "            out_file.write('' + '\\t' + '' '\\t' + meas_id + '\\t' + '' + '\\t' + 'http://eol.org/schema/terms/SampleSize' + '\\t' + str(n) + '\\n')\n",
    "            out_file_c.write(occur_id + '\\t' + t_id + '\\n')\n",
    "            out_file_t.write(t_id + '\\t' + taxon + '\\t' + par_id + '\\t' + 'species' + '\\n')\n",
    "            counter += 1\n",
    "            #we need to keep a deduplicated list of the higher taxa so we can add them to the tb_taxon.txt file. Note that blanks are skipped.\n",
    "            if genus in genera or genus == '':\n",
    "                pass\n",
    "            else:\n",
    "                genera.append(genus)\n",
    "            if family in families or family == '':\n",
    "                pass\n",
    "            else:\n",
    "                families.append(family)\n",
    "            if order in orders or order == '':\n",
    "                pass\n",
    "            else:\n",
    "                orders.append(order)\n",
    "            if class_ in classes or class_ == '':\n",
    "                pass\n",
    "            else:\n",
    "                classes.append(class_)\n",
    "            if phylum in phyla or phylum == '':\n",
    "                pass\n",
    "            else:\n",
    "                phyla.append(phylum)\n",
    "            if kingdom in kingdoms or kingdom == '':\n",
    "                pass\n",
    "            else:\n",
    "                kingdoms.append(kingdom)\n",
    "print('total taxa ' + str(len(taxa))) #this prints the number of species measurements in the tb_measurements.txt file\n",
    "#this is where we add the higher taxa to the tb_taxon.txt file\n",
    "for genus in genera:\n",
    "    g_id = ids[genus]\n",
    "    p_id = parents[genus]\n",
    "    out_file_t.write(g_id + '\\t' + genus + '\\t' + p_id + '\\t' + 'genus' + '\\n')\n",
    "for family in families:\n",
    "    f_id = ids[family]\n",
    "    p_id = parents[family]\n",
    "    out_file_t.write(f_id + '\\t' + family + '\\t' + p_id + '\\t' + '' + '\\n')\n",
    "for order in orders:\n",
    "    o_id = ids[order]\n",
    "    p_id = parents[order]\n",
    "    out_file_t.write(o_id + '\\t' + order + '\\t' + p_id + '\\t' + '' + '\\n')\n",
    "for class_ in classes:\n",
    "    c_id = ids[class_]\n",
    "    p_id = parents[class_]\n",
    "    out_file_t.write(c_id + '\\t' + class_ + '\\t' + p_id + '\\t' + '' + '\\n')\n",
    "for phylum in phyla:\n",
    "    ph_id = ids[phylum]\n",
    "    p_id = parents[phylum]\n",
    "    out_file_t.write(ph_id + '\\t' + phylum + '\\t' + p_id + '\\t' + '' + '\\n')\n",
    "for kingdom in kingdoms:\n",
    "    k_id = ids[kingdom]\n",
    "    p_id = ''\n",
    "    out_file_t.write(k_id + '\\t' + kingdom + '\\t' + p_id + '\\t' + '' + '\\n')\n",
    "print('total records ' + str(record_total))\n",
    "print('complete') #make sure code gets to end\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}