{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text tokenization\n",
    "\n",
    "This section contains code that will tokenize the transcription data and add new columns to the data frames for each transcription dataset.\n",
    "\n",
    "First, we run the definitions step from the previous section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run 02_definitions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load each transcription dataset into a data frame\n",
    "The `load_csv` function will read the data from each path constant and store data in a Pandas data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = load_csv(ANTHONY)\n",
    "c = load_csv(CATT)\n",
    "s = load_csv(STANTON)\n",
    "t = load_csv(TERRELL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optional: Preview the first five lines of a loaded dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five lines for Anthony dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-1</td>\n",
       "      <td>179295</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Susan B. Anthony      SPEECHES AND WRITINGS FI...</td>\n",
       "      <td>May 1852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-2</td>\n",
       "      <td>179296</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>/52\\r\\nS.B.A-\\r\\n\\r\\nDelivered for the\\r\\nFirs...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-3</td>\n",
       "      <td>179297</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>will the best &amp; wisest of mothers continue\\r\\n...</td>\n",
       "      <td>temperance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-4</td>\n",
       "      <td>179298</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>[Mind] the youthful mind.  Of how\\r\\nlittle av...</td>\n",
       "      <td>temperance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-5</td>\n",
       "      <td>179299</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>x\\r\\nWhile we labor to reclaim one generation ...</td>\n",
       "      <td>temperance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Campaign                      Project  \\\n",
       "0  Susan B. Anthony Papers  Speeches and other writings   \n",
       "1  Susan B. Anthony Papers  Speeches and other writings   \n",
       "2  Susan B. Anthony Papers  Speeches and other writings   \n",
       "3  Susan B. Anthony Papers  Speeches and other writings   \n",
       "4  Susan B. Anthony Papers  Speeches and other writings   \n",
       "\n",
       "                                                Item       ItemId  \\\n",
       "0  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "1  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "2  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "3  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "4  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "\n",
       "           Asset AssetId AssetStatus  \\\n",
       "0  mss11049038-1  179295   completed   \n",
       "1  mss11049038-2  179296   completed   \n",
       "2  mss11049038-3  179297   completed   \n",
       "3  mss11049038-4  179298   completed   \n",
       "4  mss11049038-5  179299   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription        Tags  \n",
       "0  Susan B. Anthony      SPEECHES AND WRITINGS FI...    May 1852  \n",
       "1  /52\\r\\nS.B.A-\\r\\n\\r\\nDelivered for the\\r\\nFirs...         NaN  \n",
       "2  will the best & wisest of mothers continue\\r\\n...  temperance  \n",
       "3  [Mind] the youthful mind.  Of how\\r\\nlittle av...  temperance  \n",
       "4  x\\r\\nWhile we labor to reclaim one generation ...  temperance  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five lines for Catt dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-1</td>\n",
       "      <td>189284</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-2</td>\n",
       "      <td>189285</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>-2-\\r\\nWe appeal in the name of our foremother...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-3</td>\n",
       "      <td>189286</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>AN APPEAL FOR LIBERTY.    1915\\r\\n\\r\\nBy Carri...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040386</td>\n",
       "      <td>mss154040386-1</td>\n",
       "      <td>189287</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040386</td>\n",
       "      <td>mss154040386-2</td>\n",
       "      <td>189288</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>The \\r\\nWoman Citizen\\r\\nA WEEKLY CHRONICLE OF...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Campaign                Project  \\\n",
       "0  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "1  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "2  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "3  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "4  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "1  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "2  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "3  Carrie Chapman Catt Papers: Speech and Article...  mss154040386   \n",
       "4  Carrie Chapman Catt Papers: Speech and Article...  mss154040386   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss154040385-1  189284   completed   \n",
       "1  mss154040385-2  189285   completed   \n",
       "2  mss154040385-3  189286   completed   \n",
       "3  mss154040386-1  189287   completed   \n",
       "4  mss154040386-2  189288   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription Tags  \n",
       "0  CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...  NaN  \n",
       "1  -2-\\r\\nWe appeal in the name of our foremother...  NaN  \n",
       "2  AN APPEAL FOR LIBERTY.    1915\\r\\n\\r\\nBy Carri...  NaN  \n",
       "3  CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...  NaN  \n",
       "4  The \\r\\nWoman Citizen\\r\\nA WEEKLY CHRONICLE OF...  NaN  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five lines for Stanton dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-1</td>\n",
       "      <td>179712</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Elizabeth Cady Stanton  GENERAL CORRESPONDENCE...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-2</td>\n",
       "      <td>179713</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>The following four letters are \\r\\nfrom Daniel...</td>\n",
       "      <td>Peter Smith; Daniel Cady; Judge Cady</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-3</td>\n",
       "      <td>179714</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>22                                            ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-4</td>\n",
       "      <td>179715</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>he could to make her respectable &amp; happy. That...</td>\n",
       "      <td>Peter Smith; Bonaparte</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-5</td>\n",
       "      <td>179716</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Johnstown 2 D     Paid 10\\r\\n\\r\\n\\r\\nPeter Smi...</td>\n",
       "      <td>Peter Smith</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Campaign                 Project  \\\n",
       "0  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "1  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "2  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "3  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "4  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "1  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "2  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "3  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "4  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss412100001-1  179712   completed   \n",
       "1  mss412100001-2  179713   completed   \n",
       "2  mss412100001-3  179714   completed   \n",
       "3  mss412100001-4  179715   completed   \n",
       "4  mss412100001-5  179716   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription  \\\n",
       "0  Elizabeth Cady Stanton  GENERAL CORRESPONDENCE...   \n",
       "1  The following four letters are \\r\\nfrom Daniel...   \n",
       "2  22                                            ...   \n",
       "3  he could to make her respectable & happy. That...   \n",
       "4  Johnstown 2 D     Paid 10\\r\\n\\r\\n\\r\\nPeter Smi...   \n",
       "\n",
       "                                   Tags  \n",
       "0                                   NaN  \n",
       "1  Peter Smith; Daniel Cady; Judge Cady  \n",
       "2                                   NaN  \n",
       "3                Peter Smith; Bonaparte  \n",
       "4                           Peter Smith  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five lines for Terrell dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-1</td>\n",
       "      <td>7580</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Office Supplies typewriter ribbons fountain pe...</td>\n",
       "      <td>Mrs Ella Wheeler Wilcox; Woman Suffrage Conven...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-2</td>\n",
       "      <td>7581</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>March 16, Wednesday,1904 - Dr. Booker Washingt...</td>\n",
       "      <td>Cruger; Calloway; VanRensselaer; Booker; Washi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-3</td>\n",
       "      <td>7582</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Fountain Pens Repaired\\r\\nTablets\\r\\nTypewrite...</td>\n",
       "      <td>Pennsylvania; committee; Washington Post</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-4</td>\n",
       "      <td>7583</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>May, 1904\\r\\n\\r\\n1 SUNDAY Received invitation ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-5</td>\n",
       "      <td>7584</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>June, 1904\\r\\n\\r\\n7 TUESDAY Reached Bremer Hav...</td>\n",
       "      <td>Berlin; Congress morning; June 1904; Paris</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Campaign  \\\n",
       "0  Mary Church Terrell: Advocate for African Amer...   \n",
       "1  Mary Church Terrell: Advocate for African Amer...   \n",
       "2  Mary Church Terrell: Advocate for African Amer...   \n",
       "3  Mary Church Terrell: Advocate for African Amer...   \n",
       "4  Mary Church Terrell: Advocate for African Amer...   \n",
       "\n",
       "                         Project  \\\n",
       "0  Address and appointment books   \n",
       "1  Address and appointment books   \n",
       "2  Address and appointment books   \n",
       "3  Address and appointment books   \n",
       "4  Address and appointment books   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "1  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "2  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "3  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "4  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss425490014-1    7580   completed   \n",
       "1  mss425490014-2    7581   completed   \n",
       "2  mss425490014-3    7582   completed   \n",
       "3  mss425490014-4    7583   completed   \n",
       "4  mss425490014-5    7584   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription  \\\n",
       "0  Office Supplies typewriter ribbons fountain pe...   \n",
       "1  March 16, Wednesday,1904 - Dr. Booker Washingt...   \n",
       "2  Fountain Pens Repaired\\r\\nTablets\\r\\nTypewrite...   \n",
       "3  May, 1904\\r\\n\\r\\n1 SUNDAY Received invitation ...   \n",
       "4  June, 1904\\r\\n\\r\\n7 TUESDAY Reached Bremer Hav...   \n",
       "\n",
       "                                                Tags  \n",
       "0  Mrs Ella Wheeler Wilcox; Woman Suffrage Conven...  \n",
       "1  Cruger; Calloway; VanRensselaer; Booker; Washi...  \n",
       "2           Pennsylvania; committee; Washington Post  \n",
       "3                                                NaN  \n",
       "4         Berlin; Congress morning; June 1904; Paris  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a new column containing the output of the `tokens` function\n",
    "The `tokens` function uses the previously loaded spaCy model to analyze each word in the transcription. This results in several values for each word, including the lemma, the part-of-speech tag, the shape of the word, and whether it is a stop word or number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenizing text for dataset: Susan B. Anthony Papers\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenizing text for dataset: Carrie Chapman Catt Papers\n",
      "Tokenizing text for dataset: Elizabeth Cady Stanton Papers\n",
      "Tokenizing text for dataset: Mary Church Terrell: Advocate for African Americans and Women\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# NOTE: This will take a while to run\n",
    "for dataset in [a, c, s, t]:\n",
    "    print(f\"Tokenizing text for dataset: {dataset['Campaign'][0]}\")\n",
    "    dataset['tokenized_text'] = dataset['Transcription'].apply(tokens)\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a new column containing the output of the `entities` function\n",
    "The `entities` function uses the previously loaded spaCy model to identify persons, places, organizations, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Identifying entities for dataset: Susan B. Anthony Papers\n",
      "Identifying entities for dataset: Carrie Chapman Catt Papers\n",
      "Identifying entities for dataset: Elizabeth Cady Stanton Papers\n",
      "Identifying entities for dataset: Mary Church Terrell: Advocate for African Americans and Women\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# NOTE: This will take a while to run\n",
    "for dataset in [a, c, s, t]:\n",
    "    print(f\"Identifying entities for dataset: {dataset['Campaign'][0]}\")\n",
    "    dataset['entities'] = dataset['Transcription'].apply(entities)\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "### Optional: Preview the results of the `entities` functions for the first row of a dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entities for first row in Anthony dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Susan B. Anthony      SPEECHES</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WRITINGS FILE        Delivered</td>\n",
       "      <td>ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>first</td>\n",
       "      <td>ORDINAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Batavia</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>CARDINAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>May 1852</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1852</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Text    Entity\n",
       "0  Susan B. Anthony      SPEECHES    PERSON\n",
       "1  WRITINGS FILE        Delivered       ORG\n",
       "2                           first   ORDINAL\n",
       "3                         Batavia       GPE\n",
       "4                               2  CARDINAL\n",
       "5                        May 1852      DATE\n",
       "6                            1852      DATE"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"Text\": row[0], \"Entity\": row[3]} for row in a['entities'].iloc[0]]).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entities for first row in Catt dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CATT</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Carrie Chapman\\r\\nSPEECH</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ARTICLE, BOOK FILE\\r\\nSpeech</td>\n",
       "      <td>LAW</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Text  Entity\n",
       "0                          CATT  PERSON\n",
       "1      Carrie Chapman\\r\\nSPEECH  PERSON\n",
       "2  ARTICLE, BOOK FILE\\r\\nSpeech     LAW"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"Text\": row[0], \"Entity\": row[3]} for row in c['entities'].iloc[0]]).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entities for first row in Stanton dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Elizabeth Cady Stanton</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1814 - 49</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Text  Entity\n",
       "0  Elizabeth Cady Stanton  PERSON\n",
       "1               1814 - 49    DATE"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"Text\": row[0], \"Entity\": row[3]} for row in s['entities'].iloc[0]]).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Entities for first row in Terrell dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Swett</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Stationery Blank Books</td>\n",
       "      <td>ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P Swett</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>February, 1904</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>178</td>\n",
       "      <td>CARDINAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Monday\\r\\n2 Tuesday\\r\\n3</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Wednesday\\r\\n4</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Thursday \\r\\n5 Friday\\r\\n6</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Crandall Association</td>\n",
       "      <td>ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7:30\\r\\nSpecial</td>\n",
       "      <td>TIME</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Text    Entity\n",
       "0                       Swett    PERSON\n",
       "1      Stationery Blank Books       ORG\n",
       "2                     P Swett    PERSON\n",
       "3              February, 1904      DATE\n",
       "4                         178  CARDINAL\n",
       "5    Monday\\r\\n2 Tuesday\\r\\n3      DATE\n",
       "6              Wednesday\\r\\n4      DATE\n",
       "7  Thursday \\r\\n5 Friday\\r\\n6      DATE\n",
       "8        Crandall Association       ORG\n",
       "9             7:30\\r\\nSpecial      TIME"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([{\"Text\": row[0], \"Entity\": row[3]} for row in t['entities'].iloc[0]]).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the `separate_text` function to isolate tokens by category\n",
    "\n",
    "The `separate_text` function uses labels generated by the `spaCy` library to organize the contents of each transcription into actual text, stop words (conjunctions, prepositions, etc.), non-alphanumeric strings (punctuation, whitespace, etc.), numbers, and ambiguous words (when a transcriber cannot make out a word or character, a `?` will be used for the unknown character(s); this is reflected in the analyzed pattern of the word which is used to remove these words from the text category)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Organizing tokens by category for: Susan B. Anthony Papers\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Organizing tokens by category for: Carrie Chapman Catt Papers\n",
      "Organizing tokens by category for: Elizabeth Cady Stanton Papers\n",
      "Organizing tokens by category for: Mary Church Terrell: Advocate for African Americans and Women\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# Run the separate_text function on the Anthony data frame\n",
    "for dataset in [a, c, s, t]:\n",
    "    print(f\"Organizing tokens by category for: {dataset['Campaign'][0]}\")\n",
    "    separate_text(dataset)\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cache the result for next steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [a, c, s, t]:\n",
    "    write_cache(dataset, str(dataset['Campaign'][0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "### Optional: Preview the results for the first five rows of the updated data frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five rows of updated Anthony dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>entities</th>\n",
       "      <th>text</th>\n",
       "      <th>stop_words</th>\n",
       "      <th>nonalphanums</th>\n",
       "      <th>numbers</th>\n",
       "      <th>ambigs</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-1</td>\n",
       "      <td>179295</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Susan B. Anthony      SPEECHES AND WRITINGS FI...</td>\n",
       "      <td>May 1852</td>\n",
       "      <td>[(Susan, Susan, PROPN, NNP, compound, Xxxxx, T...</td>\n",
       "      <td>[(Susan B. Anthony      SPEECHES, 0, 30, PERSO...</td>\n",
       "      <td>[(Susan, Susan, PROPN, NNP, compound, Xxxxx, T...</td>\n",
       "      <td>[(AND, and, CCONJ, CC, cc, XXX, True, True), (...</td>\n",
       "      <td>[(     ,      , SPACE, _SP, dep,     , False, ...</td>\n",
       "      <td>[(2, 2, NUM, CD, nummod, d, False, False), (18...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[susan, b., anthony, speeches, writing, file, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-2</td>\n",
       "      <td>179296</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>/52\\r\\nS.B.A-\\r\\n\\r\\nDelivered for the\\r\\nFirs...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(/52, /52, PROPN, NNP, punct, /dd, False, Fal...</td>\n",
       "      <td>[(Batavia, 44, 51, GPE), (N.J., 52, 56, GPE), ...</td>\n",
       "      <td>[(/52, /52, PROPN, NNP, punct, /dd, False, Fal...</td>\n",
       "      <td>[(for, for, ADP, IN, prep, xxx, True, True), (...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[(1852, 1852, NUM, CD, nummod, dddd, False, Fa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[/52, s.b.a-, deliver, batavia, n.j., company,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-3</td>\n",
       "      <td>179297</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>will the best &amp; wisest of mothers continue\\r\\n...</td>\n",
       "      <td>temperance</td>\n",
       "      <td>[(will, will, AUX, MD, aux, xxxx, True, True),...</td>\n",
       "      <td>[(the\\r\\nSociety, 295, 307, ORG), (two, 324, 3...</td>\n",
       "      <td>[(best, good, ADJ, JJS, nsubj, xxxx, True, Fal...</td>\n",
       "      <td>[(will, will, AUX, MD, aux, xxxx, True, True),...</td>\n",
       "      <td>[(&amp;, &amp;, CCONJ, CC, cc, &amp;, False, False), (\\r\\n...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[good, wise, mother, continue, son, fall, vict...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-4</td>\n",
       "      <td>179298</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>[Mind] the youthful mind.  Of how\\r\\nlittle av...</td>\n",
       "      <td>temperance</td>\n",
       "      <td>[([, [, X, XX, dep, [, False, False), (Mind, m...</td>\n",
       "      <td>[(christian, 77, 86, NORP), (truth  &amp; sobernes...</td>\n",
       "      <td>[(Mind, mind, VERB, VB, dep, Xxxx, True, False...</td>\n",
       "      <td>[(the, the, DET, DT, det, xxx, True, True), (O...</td>\n",
       "      <td>[([, [, X, XX, dep, [, False, False), (], ], X...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[mind, youthful, mind, little, avail, untire, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Susan B. Anthony Papers</td>\n",
       "      <td>Speeches and other writings</td>\n",
       "      <td>Susan B. Anthony Papers: Speeches and Writings...</td>\n",
       "      <td>mss11049038</td>\n",
       "      <td>mss11049038-5</td>\n",
       "      <td>179299</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>x\\r\\nWhile we labor to reclaim one generation ...</td>\n",
       "      <td>temperance</td>\n",
       "      <td>[(x, x, ADP, IN, punct, x, True, False), (\\r\\n...</td>\n",
       "      <td>[(one, 29, 32, CARDINAL), (Legislature, 145, 1...</td>\n",
       "      <td>[(x, x, ADP, IN, punct, x, True, False), (labo...</td>\n",
       "      <td>[(While, while, SCONJ, IN, mark, Xxxxx, True, ...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[x, labor, reclaim, generation, drunkard, rise...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Campaign                      Project  \\\n",
       "0  Susan B. Anthony Papers  Speeches and other writings   \n",
       "1  Susan B. Anthony Papers  Speeches and other writings   \n",
       "2  Susan B. Anthony Papers  Speeches and other writings   \n",
       "3  Susan B. Anthony Papers  Speeches and other writings   \n",
       "4  Susan B. Anthony Papers  Speeches and other writings   \n",
       "\n",
       "                                                Item       ItemId  \\\n",
       "0  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "1  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "2  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "3  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "4  Susan B. Anthony Papers: Speeches and Writings...  mss11049038   \n",
       "\n",
       "           Asset AssetId AssetStatus  \\\n",
       "0  mss11049038-1  179295   completed   \n",
       "1  mss11049038-2  179296   completed   \n",
       "2  mss11049038-3  179297   completed   \n",
       "3  mss11049038-4  179298   completed   \n",
       "4  mss11049038-5  179299   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription        Tags  \\\n",
       "0  Susan B. Anthony      SPEECHES AND WRITINGS FI...    May 1852   \n",
       "1  /52\\r\\nS.B.A-\\r\\n\\r\\nDelivered for the\\r\\nFirs...         NaN   \n",
       "2  will the best & wisest of mothers continue\\r\\n...  temperance   \n",
       "3  [Mind] the youthful mind.  Of how\\r\\nlittle av...  temperance   \n",
       "4  x\\r\\nWhile we labor to reclaim one generation ...  temperance   \n",
       "\n",
       "                                      tokenized_text  \\\n",
       "0  [(Susan, Susan, PROPN, NNP, compound, Xxxxx, T...   \n",
       "1  [(/52, /52, PROPN, NNP, punct, /dd, False, Fal...   \n",
       "2  [(will, will, AUX, MD, aux, xxxx, True, True),...   \n",
       "3  [([, [, X, XX, dep, [, False, False), (Mind, m...   \n",
       "4  [(x, x, ADP, IN, punct, x, True, False), (\\r\\n...   \n",
       "\n",
       "                                            entities  \\\n",
       "0  [(Susan B. Anthony      SPEECHES, 0, 30, PERSO...   \n",
       "1  [(Batavia, 44, 51, GPE), (N.J., 52, 56, GPE), ...   \n",
       "2  [(the\\r\\nSociety, 295, 307, ORG), (two, 324, 3...   \n",
       "3  [(christian, 77, 86, NORP), (truth  & sobernes...   \n",
       "4  [(one, 29, 32, CARDINAL), (Legislature, 145, 1...   \n",
       "\n",
       "                                                text  \\\n",
       "0  [(Susan, Susan, PROPN, NNP, compound, Xxxxx, T...   \n",
       "1  [(/52, /52, PROPN, NNP, punct, /dd, False, Fal...   \n",
       "2  [(best, good, ADJ, JJS, nsubj, xxxx, True, Fal...   \n",
       "3  [(Mind, mind, VERB, VB, dep, Xxxx, True, False...   \n",
       "4  [(x, x, ADP, IN, punct, x, True, False), (labo...   \n",
       "\n",
       "                                          stop_words  \\\n",
       "0  [(AND, and, CCONJ, CC, cc, XXX, True, True), (...   \n",
       "1  [(for, for, ADP, IN, prep, xxx, True, True), (...   \n",
       "2  [(will, will, AUX, MD, aux, xxxx, True, True),...   \n",
       "3  [(the, the, DET, DT, det, xxx, True, True), (O...   \n",
       "4  [(While, while, SCONJ, IN, mark, Xxxxx, True, ...   \n",
       "\n",
       "                                        nonalphanums  \\\n",
       "0  [(     ,      , SPACE, _SP, dep,     , False, ...   \n",
       "1  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "2  [(&, &, CCONJ, CC, cc, &, False, False), (\\r\\n...   \n",
       "3  [([, [, X, XX, dep, [, False, False), (], ], X...   \n",
       "4  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "\n",
       "                                             numbers ambigs  \\\n",
       "0  [(2, 2, NUM, CD, nummod, d, False, False), (18...     []   \n",
       "1  [(1852, 1852, NUM, CD, nummod, dddd, False, Fa...     []   \n",
       "2                                                 []     []   \n",
       "3                                                 []     []   \n",
       "4                                                 []     []   \n",
       "\n",
       "                                      processed_text  \n",
       "0  [susan, b., anthony, speeches, writing, file, ...  \n",
       "1  [/52, s.b.a-, deliver, batavia, n.j., company,...  \n",
       "2  [good, wise, mother, continue, son, fall, vict...  \n",
       "3  [mind, youthful, mind, little, avail, untire, ...  \n",
       "4  [x, labor, reclaim, generation, drunkard, rise...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.iloc[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five rows of updated Catt dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>entities</th>\n",
       "      <th>text</th>\n",
       "      <th>stop_words</th>\n",
       "      <th>nonalphanums</th>\n",
       "      <th>numbers</th>\n",
       "      <th>ambigs</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-1</td>\n",
       "      <td>189284</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...</td>\n",
       "      <td>[(CATT, 0, 4, PERSON), (Carrie Chapman\\r\\nSPEE...</td>\n",
       "      <td>[(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...</td>\n",
       "      <td>[(An, an, DET, DT, det, Xx, True, True), (For,...</td>\n",
       "      <td>[(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[catt, carrie, chapman, speech, article, book,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-2</td>\n",
       "      <td>189285</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>-2-\\r\\nWe appeal in the name of our foremother...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(-2-, -2-, PUNCT, ``, punct, -d-, False, Fals...</td>\n",
       "      <td>[(-2-\\r\\n, 0, 5, PERSON), (American, 428, 436,...</td>\n",
       "      <td>[(appeal, appeal, VERB, VBP, ccomp, xxxx, True...</td>\n",
       "      <td>[(We, we, PRON, PRP, nsubj, Xx, True, True), (...</td>\n",
       "      <td>[(-2-, -2-, PUNCT, ``, punct, -d-, False, Fals...</td>\n",
       "      <td>[(1,600,000, 1,600,000, NUM, CD, nummod, d,ddd...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[appeal, foremother, forefather, equal, courag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040385</td>\n",
       "      <td>mss154040385-3</td>\n",
       "      <td>189286</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>AN APPEAL FOR LIBERTY.    1915\\r\\n\\r\\nBy Carri...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(AN, an, DET, DT, det, XX, True, True), (APPE...</td>\n",
       "      <td>[(1915, 26, 30, DATE), (Carrie Chapman Catt, 3...</td>\n",
       "      <td>[(APPEAL, APPEAL, PROPN, NNP, ROOT, XXXX, True...</td>\n",
       "      <td>[(AN, an, DET, DT, det, XX, True, True), (FOR,...</td>\n",
       "      <td>[(., ., PUNCT, ., punct, ., False, False), (  ...</td>\n",
       "      <td>[(1915, 1915, NUM, CD, ROOT, dddd, False, Fals...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[appeal, liberty, carrie, chapman, catt, year,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040386</td>\n",
       "      <td>mss154040386-1</td>\n",
       "      <td>189287</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...</td>\n",
       "      <td>[(CATT, 0, 4, PERSON), (Carrie Chapman\\r\\nSPEE...</td>\n",
       "      <td>[(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...</td>\n",
       "      <td>[(Be, be, AUX, VB, ROOT, Xx, True, True)]</td>\n",
       "      <td>[(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[catt, carrie, chapman, speech, article, book,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Carrie Chapman Catt Papers</td>\n",
       "      <td>Speeches and articles</td>\n",
       "      <td>Carrie Chapman Catt Papers: Speech and Article...</td>\n",
       "      <td>mss154040386</td>\n",
       "      <td>mss154040386-2</td>\n",
       "      <td>189288</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>The \\r\\nWoman Citizen\\r\\nA WEEKLY CHRONICLE OF...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(The, the, DET, DT, det, Xxx, True, True), (\\...</td>\n",
       "      <td>[(Carrie Chapman Catt, 156, 175, PERSON), (Con...</td>\n",
       "      <td>[(Woman, Woman, PROPN, NNP, compound, Xxxxx, T...</td>\n",
       "      <td>[(The, the, DET, DT, det, Xxx, True, True), (A...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[(21, 21, NUM, CD, nummod, dd, False, False), ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[woman, citizen, weekly, chronicle, progress, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Campaign                Project  \\\n",
       "0  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "1  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "2  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "3  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "4  Carrie Chapman Catt Papers  Speeches and articles   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "1  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "2  Carrie Chapman Catt Papers: Speech and Article...  mss154040385   \n",
       "3  Carrie Chapman Catt Papers: Speech and Article...  mss154040386   \n",
       "4  Carrie Chapman Catt Papers: Speech and Article...  mss154040386   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss154040385-1  189284   completed   \n",
       "1  mss154040385-2  189285   completed   \n",
       "2  mss154040385-3  189286   completed   \n",
       "3  mss154040386-1  189287   completed   \n",
       "4  mss154040386-2  189288   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription Tags  \\\n",
       "0  CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...  NaN   \n",
       "1  -2-\\r\\nWe appeal in the name of our foremother...  NaN   \n",
       "2  AN APPEAL FOR LIBERTY.    1915\\r\\n\\r\\nBy Carri...  NaN   \n",
       "3  CATT, Carrie Chapman\\r\\nSPEECH, ARTICLE, BOOK ...  NaN   \n",
       "4  The \\r\\nWoman Citizen\\r\\nA WEEKLY CHRONICLE OF...  NaN   \n",
       "\n",
       "                                      tokenized_text  \\\n",
       "0  [(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...   \n",
       "1  [(-2-, -2-, PUNCT, ``, punct, -d-, False, Fals...   \n",
       "2  [(AN, an, DET, DT, det, XX, True, True), (APPE...   \n",
       "3  [(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...   \n",
       "4  [(The, the, DET, DT, det, Xxx, True, True), (\\...   \n",
       "\n",
       "                                            entities  \\\n",
       "0  [(CATT, 0, 4, PERSON), (Carrie Chapman\\r\\nSPEE...   \n",
       "1  [(-2-\\r\\n, 0, 5, PERSON), (American, 428, 436,...   \n",
       "2  [(1915, 26, 30, DATE), (Carrie Chapman Catt, 3...   \n",
       "3  [(CATT, 0, 4, PERSON), (Carrie Chapman\\r\\nSPEE...   \n",
       "4  [(Carrie Chapman Catt, 156, 175, PERSON), (Con...   \n",
       "\n",
       "                                                text  \\\n",
       "0  [(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...   \n",
       "1  [(appeal, appeal, VERB, VBP, ccomp, xxxx, True...   \n",
       "2  [(APPEAL, APPEAL, PROPN, NNP, ROOT, XXXX, True...   \n",
       "3  [(CATT, CATT, PROPN, NNP, ROOT, XXXX, True, Fa...   \n",
       "4  [(Woman, Woman, PROPN, NNP, compound, Xxxxx, T...   \n",
       "\n",
       "                                          stop_words  \\\n",
       "0  [(An, an, DET, DT, det, Xx, True, True), (For,...   \n",
       "1  [(We, we, PRON, PRP, nsubj, Xx, True, True), (...   \n",
       "2  [(AN, an, DET, DT, det, XX, True, True), (FOR,...   \n",
       "3          [(Be, be, AUX, VB, ROOT, Xx, True, True)]   \n",
       "4  [(The, the, DET, DT, det, Xxx, True, True), (A...   \n",
       "\n",
       "                                        nonalphanums  \\\n",
       "0  [(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...   \n",
       "1  [(-2-, -2-, PUNCT, ``, punct, -d-, False, Fals...   \n",
       "2  [(., ., PUNCT, ., punct, ., False, False), (  ...   \n",
       "3  [(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...   \n",
       "4  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "\n",
       "                                             numbers ambigs  \\\n",
       "0                                                 []     []   \n",
       "1  [(1,600,000, 1,600,000, NUM, CD, nummod, d,ddd...     []   \n",
       "2  [(1915, 1915, NUM, CD, ROOT, dddd, False, Fals...     []   \n",
       "3                                                 []     []   \n",
       "4  [(21, 21, NUM, CD, nummod, dd, False, False), ...     []   \n",
       "\n",
       "                                      processed_text  \n",
       "0  [catt, carrie, chapman, speech, article, book,...  \n",
       "1  [appeal, foremother, forefather, equal, courag...  \n",
       "2  [appeal, liberty, carrie, chapman, catt, year,...  \n",
       "3  [catt, carrie, chapman, speech, article, book,...  \n",
       "4  [woman, citizen, weekly, chronicle, progress, ...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.iloc[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five rows of updated Stanton dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>entities</th>\n",
       "      <th>text</th>\n",
       "      <th>stop_words</th>\n",
       "      <th>nonalphanums</th>\n",
       "      <th>numbers</th>\n",
       "      <th>ambigs</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-1</td>\n",
       "      <td>179712</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Elizabeth Cady Stanton  GENERAL CORRESPONDENCE...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(Elizabeth, Elizabeth, PROPN, NNP, compound, ...</td>\n",
       "      <td>[(Elizabeth Cady Stanton, 0, 22, PERSON), (181...</td>\n",
       "      <td>[(Elizabeth, Elizabeth, PROPN, NNP, compound, ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[( ,  , SPACE, _SP, dep,  , False, False), (-,...</td>\n",
       "      <td>[(1814, 1814, NUM, CD, appos, dddd, False, Fal...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[elizabeth, cady, stanton, general, correspond...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-2</td>\n",
       "      <td>179713</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>The following four letters are \\r\\nfrom Daniel...</td>\n",
       "      <td>Peter Smith; Daniel Cady; Judge Cady</td>\n",
       "      <td>[(The, the, DET, DT, det, Xxx, True, True), (f...</td>\n",
       "      <td>[(four, 14, 18, CARDINAL), (Daniel Cady, 38, 4...</td>\n",
       "      <td>[(following, follow, VERB, VBG, amod, xxxx, Tr...</td>\n",
       "      <td>[(The, the, DET, DT, det, Xxx, True, True), (f...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[follow, letter, daniel, cady, peter, smith, j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-3</td>\n",
       "      <td>179714</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>22                                            ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(22, 22, NUM, CD, ROOT, dd, False, False), ( ...</td>\n",
       "      <td>[(22, 0, 2, CARDINAL), (2 Dec. 1814, 91, 102, ...</td>\n",
       "      <td>[(Dec., Dec., PROPN, NNP, npadvmod, Xxx., Fals...</td>\n",
       "      <td>[(It, it, PRON, PRP, nsubj, Xx, True, True), (...</td>\n",
       "      <td>[(                                            ...</td>\n",
       "      <td>[(22, 22, NUM, CD, ROOT, dd, False, False), (2...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[dec., dear, sir, true, lose, young, child, th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-4</td>\n",
       "      <td>179715</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>he could to make her respectable &amp; happy. That...</td>\n",
       "      <td>Peter Smith; Bonaparte</td>\n",
       "      <td>[(he, he, PRON, PRP, nsubj, xx, True, True), (...</td>\n",
       "      <td>[(one, 467, 470, CARDINAL), (one, 614, 617, CA...</td>\n",
       "      <td>[(respectable, respectable, ADJ, JJ, ccomp, xx...</td>\n",
       "      <td>[(he, he, PRON, PRP, nsubj, xx, True, True), (...</td>\n",
       "      <td>[(&amp;, &amp;, CCONJ, CC, cc, &amp;, False, False), (., ....</td>\n",
       "      <td>[(2d, 2d, NUM, CD, nummod, dx, False, False), ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[respectable, happy, moment, flatter, soon, se...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Elizabeth Cady Stanton Papers</td>\n",
       "      <td>General correspondence</td>\n",
       "      <td>Elizabeth Cady Stanton Papers: General Corresp...</td>\n",
       "      <td>mss412100001</td>\n",
       "      <td>mss412100001-5</td>\n",
       "      <td>179716</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Johnstown 2 D     Paid 10\\r\\n\\r\\n\\r\\nPeter Smi...</td>\n",
       "      <td>Peter Smith</td>\n",
       "      <td>[(Johnstown, Johnstown, PROPN, NNP, nmod, Xxxx...</td>\n",
       "      <td>[(Johnstown, 0, 9, GPE), (10, 23, 25, CARDINAL...</td>\n",
       "      <td>[(Johnstown, Johnstown, PROPN, NNP, nmod, Xxxx...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[(    ,     , SPACE, _SP, dep,     , False, Fa...</td>\n",
       "      <td>[(2, 2, NUM, CD, nummod, d, False, False), (10...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[johnstown, d, paid, peter, smith, esquire, pe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Campaign                 Project  \\\n",
       "0  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "1  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "2  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "3  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "4  Elizabeth Cady Stanton Papers  General correspondence   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "1  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "2  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "3  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "4  Elizabeth Cady Stanton Papers: General Corresp...  mss412100001   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss412100001-1  179712   completed   \n",
       "1  mss412100001-2  179713   completed   \n",
       "2  mss412100001-3  179714   completed   \n",
       "3  mss412100001-4  179715   completed   \n",
       "4  mss412100001-5  179716   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription  \\\n",
       "0  Elizabeth Cady Stanton  GENERAL CORRESPONDENCE...   \n",
       "1  The following four letters are \\r\\nfrom Daniel...   \n",
       "2  22                                            ...   \n",
       "3  he could to make her respectable & happy. That...   \n",
       "4  Johnstown 2 D     Paid 10\\r\\n\\r\\n\\r\\nPeter Smi...   \n",
       "\n",
       "                                   Tags  \\\n",
       "0                                   NaN   \n",
       "1  Peter Smith; Daniel Cady; Judge Cady   \n",
       "2                                   NaN   \n",
       "3                Peter Smith; Bonaparte   \n",
       "4                           Peter Smith   \n",
       "\n",
       "                                      tokenized_text  \\\n",
       "0  [(Elizabeth, Elizabeth, PROPN, NNP, compound, ...   \n",
       "1  [(The, the, DET, DT, det, Xxx, True, True), (f...   \n",
       "2  [(22, 22, NUM, CD, ROOT, dd, False, False), ( ...   \n",
       "3  [(he, he, PRON, PRP, nsubj, xx, True, True), (...   \n",
       "4  [(Johnstown, Johnstown, PROPN, NNP, nmod, Xxxx...   \n",
       "\n",
       "                                            entities  \\\n",
       "0  [(Elizabeth Cady Stanton, 0, 22, PERSON), (181...   \n",
       "1  [(four, 14, 18, CARDINAL), (Daniel Cady, 38, 4...   \n",
       "2  [(22, 0, 2, CARDINAL), (2 Dec. 1814, 91, 102, ...   \n",
       "3  [(one, 467, 470, CARDINAL), (one, 614, 617, CA...   \n",
       "4  [(Johnstown, 0, 9, GPE), (10, 23, 25, CARDINAL...   \n",
       "\n",
       "                                                text  \\\n",
       "0  [(Elizabeth, Elizabeth, PROPN, NNP, compound, ...   \n",
       "1  [(following, follow, VERB, VBG, amod, xxxx, Tr...   \n",
       "2  [(Dec., Dec., PROPN, NNP, npadvmod, Xxx., Fals...   \n",
       "3  [(respectable, respectable, ADJ, JJ, ccomp, xx...   \n",
       "4  [(Johnstown, Johnstown, PROPN, NNP, nmod, Xxxx...   \n",
       "\n",
       "                                          stop_words  \\\n",
       "0                                                 []   \n",
       "1  [(The, the, DET, DT, det, Xxx, True, True), (f...   \n",
       "2  [(It, it, PRON, PRP, nsubj, Xx, True, True), (...   \n",
       "3  [(he, he, PRON, PRP, nsubj, xx, True, True), (...   \n",
       "4                                                 []   \n",
       "\n",
       "                                        nonalphanums  \\\n",
       "0  [( ,  , SPACE, _SP, dep,  , False, False), (-,...   \n",
       "1  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "2  [(                                            ...   \n",
       "3  [(&, &, CCONJ, CC, cc, &, False, False), (., ....   \n",
       "4  [(    ,     , SPACE, _SP, dep,     , False, Fa...   \n",
       "\n",
       "                                             numbers ambigs  \\\n",
       "0  [(1814, 1814, NUM, CD, appos, dddd, False, Fal...     []   \n",
       "1                                                 []     []   \n",
       "2  [(22, 22, NUM, CD, ROOT, dd, False, False), (2...     []   \n",
       "3  [(2d, 2d, NUM, CD, nummod, dx, False, False), ...     []   \n",
       "4  [(2, 2, NUM, CD, nummod, d, False, False), (10...     []   \n",
       "\n",
       "                                      processed_text  \n",
       "0  [elizabeth, cady, stanton, general, correspond...  \n",
       "1  [follow, letter, daniel, cady, peter, smith, j...  \n",
       "2  [dec., dear, sir, true, lose, young, child, th...  \n",
       "3  [respectable, happy, moment, flatter, soon, se...  \n",
       "4  [johnstown, d, paid, peter, smith, esquire, pe...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.iloc[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First five rows of updated Terrell dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Campaign</th>\n",
       "      <th>Project</th>\n",
       "      <th>Item</th>\n",
       "      <th>ItemId</th>\n",
       "      <th>Asset</th>\n",
       "      <th>AssetId</th>\n",
       "      <th>AssetStatus</th>\n",
       "      <th>DownloadUrl</th>\n",
       "      <th>Transcription</th>\n",
       "      <th>Tags</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>entities</th>\n",
       "      <th>text</th>\n",
       "      <th>stop_words</th>\n",
       "      <th>nonalphanums</th>\n",
       "      <th>numbers</th>\n",
       "      <th>ambigs</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-1</td>\n",
       "      <td>7580</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Office Supplies typewriter ribbons fountain pe...</td>\n",
       "      <td>Mrs Ella Wheeler Wilcox; Woman Suffrage Conven...</td>\n",
       "      <td>[(Office, office, NOUN, NN, compound, Xxxxx, T...</td>\n",
       "      <td>[(Swett, 101, 106, PERSON), (Stationery Blank ...</td>\n",
       "      <td>[(Office, office, NOUN, NN, compound, Xxxxx, T...</td>\n",
       "      <td>[(’s, ’s, PART, POS, case, ’x, False, True), (...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[(603, 603, NUM, CD, nummod, ddd, False, False...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[office, supply, typewriter, ribbon, fountain,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-2</td>\n",
       "      <td>7581</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>March 16, Wednesday,1904 - Dr. Booker Washingt...</td>\n",
       "      <td>Cruger; Calloway; VanRensselaer; Booker; Washi...</td>\n",
       "      <td>[(March, March, PROPN, NNP, npadvmod, Xxxxx, T...</td>\n",
       "      <td>[(March 16, 0, 8, DATE), (Booker, 31, 37, PERS...</td>\n",
       "      <td>[(March, March, PROPN, NNP, npadvmod, Xxxxx, T...</td>\n",
       "      <td>[(as, as, ADP, IN, prep, xx, True, True), (our...</td>\n",
       "      <td>[(,, ,, PUNCT, ,, punct, ,, False, False), (-,...</td>\n",
       "      <td>[(16, 16, NUM, CD, nummod, dd, False, False), ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[march, wednesday,1904, dr., booker, washingto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-3</td>\n",
       "      <td>7582</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>Fountain Pens Repaired\\r\\nTablets\\r\\nTypewrite...</td>\n",
       "      <td>Pennsylvania; committee; Washington Post</td>\n",
       "      <td>[(Fountain, Fountain, PROPN, NNP, compound, Xx...</td>\n",
       "      <td>[(Fountain Pens Repaired\\r\\nTablets\\r\\nTypewri...</td>\n",
       "      <td>[(Fountain, Fountain, PROPN, NNP, compound, Xx...</td>\n",
       "      <td>[('s, 's, PART, POS, case, 'x, False, True), (...</td>\n",
       "      <td>[(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...</td>\n",
       "      <td>[(603, 603, NUM, CD, nummod, ddd, False, False...</td>\n",
       "      <td>[(?, ?, ADJ, JJ, punct, ?, False, False), (Wi?...</td>\n",
       "      <td>[fountain, pens, repaired, tablet, typewriter,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-4</td>\n",
       "      <td>7583</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>May, 1904\\r\\n\\r\\n1 SUNDAY Received invitation ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[(May, May, PROPN, NNP, nmod, Xxx, True, True)...</td>\n",
       "      <td>[(May, 1904, 0, 9, DATE), (1, 13, 14, CARDINAL...</td>\n",
       "      <td>[(SUNDAY, SUNDAY, PROPN, NNP, appos, XXXX, Tru...</td>\n",
       "      <td>[(May, May, PROPN, NNP, nmod, Xxx, True, True)...</td>\n",
       "      <td>[(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...</td>\n",
       "      <td>[(1904, 1904, NUM, CD, nummod, dddd, False, Fa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[sunday, receive, invitation, fran, olga, mr, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mary Church Terrell: Advocate for African Amer...</td>\n",
       "      <td>Address and appointment books</td>\n",
       "      <td>Mary Church Terrell Papers: Appointment Calend...</td>\n",
       "      <td>mss425490014</td>\n",
       "      <td>mss425490014-5</td>\n",
       "      <td>7584</td>\n",
       "      <td>completed</td>\n",
       "      <td>http://tile.loc.gov/image-services/iiif/servic...</td>\n",
       "      <td>June, 1904\\r\\n\\r\\n7 TUESDAY Reached Bremer Hav...</td>\n",
       "      <td>Berlin; Congress morning; June 1904; Paris</td>\n",
       "      <td>[(June, June, PROPN, NNP, npadvmod, Xxxx, True...</td>\n",
       "      <td>[(June, 1904, 0, 10, DATE), (7, 14, 15, CARDIN...</td>\n",
       "      <td>[(June, June, PROPN, NNP, npadvmod, Xxxx, True...</td>\n",
       "      <td>[(in, in, ADP, IN, prep, xx, True, True), (at,...</td>\n",
       "      <td>[(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...</td>\n",
       "      <td>[(1904, 1904, NUM, CD, nummod, dddd, False, Fa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[june, tuesday, reach, bremer, haven, morning,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Campaign  \\\n",
       "0  Mary Church Terrell: Advocate for African Amer...   \n",
       "1  Mary Church Terrell: Advocate for African Amer...   \n",
       "2  Mary Church Terrell: Advocate for African Amer...   \n",
       "3  Mary Church Terrell: Advocate for African Amer...   \n",
       "4  Mary Church Terrell: Advocate for African Amer...   \n",
       "\n",
       "                         Project  \\\n",
       "0  Address and appointment books   \n",
       "1  Address and appointment books   \n",
       "2  Address and appointment books   \n",
       "3  Address and appointment books   \n",
       "4  Address and appointment books   \n",
       "\n",
       "                                                Item        ItemId  \\\n",
       "0  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "1  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "2  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "3  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "4  Mary Church Terrell Papers: Appointment Calend...  mss425490014   \n",
       "\n",
       "            Asset AssetId AssetStatus  \\\n",
       "0  mss425490014-1    7580   completed   \n",
       "1  mss425490014-2    7581   completed   \n",
       "2  mss425490014-3    7582   completed   \n",
       "3  mss425490014-4    7583   completed   \n",
       "4  mss425490014-5    7584   completed   \n",
       "\n",
       "                                         DownloadUrl  \\\n",
       "0  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "1  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "2  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "3  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "4  http://tile.loc.gov/image-services/iiif/servic...   \n",
       "\n",
       "                                       Transcription  \\\n",
       "0  Office Supplies typewriter ribbons fountain pe...   \n",
       "1  March 16, Wednesday,1904 - Dr. Booker Washingt...   \n",
       "2  Fountain Pens Repaired\\r\\nTablets\\r\\nTypewrite...   \n",
       "3  May, 1904\\r\\n\\r\\n1 SUNDAY Received invitation ...   \n",
       "4  June, 1904\\r\\n\\r\\n7 TUESDAY Reached Bremer Hav...   \n",
       "\n",
       "                                                Tags  \\\n",
       "0  Mrs Ella Wheeler Wilcox; Woman Suffrage Conven...   \n",
       "1  Cruger; Calloway; VanRensselaer; Booker; Washi...   \n",
       "2           Pennsylvania; committee; Washington Post   \n",
       "3                                                NaN   \n",
       "4         Berlin; Congress morning; June 1904; Paris   \n",
       "\n",
       "                                      tokenized_text  \\\n",
       "0  [(Office, office, NOUN, NN, compound, Xxxxx, T...   \n",
       "1  [(March, March, PROPN, NNP, npadvmod, Xxxxx, T...   \n",
       "2  [(Fountain, Fountain, PROPN, NNP, compound, Xx...   \n",
       "3  [(May, May, PROPN, NNP, nmod, Xxx, True, True)...   \n",
       "4  [(June, June, PROPN, NNP, npadvmod, Xxxx, True...   \n",
       "\n",
       "                                            entities  \\\n",
       "0  [(Swett, 101, 106, PERSON), (Stationery Blank ...   \n",
       "1  [(March 16, 0, 8, DATE), (Booker, 31, 37, PERS...   \n",
       "2  [(Fountain Pens Repaired\\r\\nTablets\\r\\nTypewri...   \n",
       "3  [(May, 1904, 0, 9, DATE), (1, 13, 14, CARDINAL...   \n",
       "4  [(June, 1904, 0, 10, DATE), (7, 14, 15, CARDIN...   \n",
       "\n",
       "                                                text  \\\n",
       "0  [(Office, office, NOUN, NN, compound, Xxxxx, T...   \n",
       "1  [(March, March, PROPN, NNP, npadvmod, Xxxxx, T...   \n",
       "2  [(Fountain, Fountain, PROPN, NNP, compound, Xx...   \n",
       "3  [(SUNDAY, SUNDAY, PROPN, NNP, appos, XXXX, Tru...   \n",
       "4  [(June, June, PROPN, NNP, npadvmod, Xxxx, True...   \n",
       "\n",
       "                                          stop_words  \\\n",
       "0  [(’s, ’s, PART, POS, case, ’x, False, True), (...   \n",
       "1  [(as, as, ADP, IN, prep, xx, True, True), (our...   \n",
       "2  [('s, 's, PART, POS, case, 'x, False, True), (...   \n",
       "3  [(May, May, PROPN, NNP, nmod, Xxx, True, True)...   \n",
       "4  [(in, in, ADP, IN, prep, xx, True, True), (at,...   \n",
       "\n",
       "                                        nonalphanums  \\\n",
       "0  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "1  [(,, ,, PUNCT, ,, punct, ,, False, False), (-,...   \n",
       "2  [(\\r\\n, \\r\\n, SPACE, _SP, dep, \\r\\n, False, Fa...   \n",
       "3  [(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...   \n",
       "4  [(,, ,, PUNCT, ,, punct, ,, False, False), (\\r...   \n",
       "\n",
       "                                             numbers  \\\n",
       "0  [(603, 603, NUM, CD, nummod, ddd, False, False...   \n",
       "1  [(16, 16, NUM, CD, nummod, dd, False, False), ...   \n",
       "2  [(603, 603, NUM, CD, nummod, ddd, False, False...   \n",
       "3  [(1904, 1904, NUM, CD, nummod, dddd, False, Fa...   \n",
       "4  [(1904, 1904, NUM, CD, nummod, dddd, False, Fa...   \n",
       "\n",
       "                                              ambigs  \\\n",
       "0                                                 []   \n",
       "1                                                 []   \n",
       "2  [(?, ?, ADJ, JJ, punct, ?, False, False), (Wi?...   \n",
       "3                                                 []   \n",
       "4                                                 []   \n",
       "\n",
       "                                      processed_text  \n",
       "0  [office, supply, typewriter, ribbon, fountain,...  \n",
       "1  [march, wednesday,1904, dr., booker, washingto...  \n",
       "2  [fountain, pens, repaired, tablet, typewriter,...  \n",
       "3  [sunday, receive, invitation, fran, olga, mr, ...  \n",
       "4  [june, tuesday, reach, bremer, haven, morning,...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.iloc[0:5]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}