02_Scrape_Billboard_top_100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape Billboard Top 100 charts from 1958-2012 to fill in missing data if a song was popular or not from original Million Song database.  \n",
    "\n",
    "## 1958 is the earliest they have on record, even though the MSDatabase goes back to 1922"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import string\n",
    "from dateutil import parser\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A function to create list of Billboard Hot 100 urls that catalogue each song going back to 1958, up until 2012 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_soup(start_url):\n",
    "    response = requests.get(start_url)\n",
    "    page = response.text\n",
    "    return BeautifulSoup(page, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from datetime import date, datetime, timedelta\n",
    "\n",
    "def perdelta(start, end, delta):\n",
    "    curr = start\n",
    "    while curr < end:\n",
    "        yield curr\n",
    "        curr += delta\n",
    "\n",
    "urls = []\n",
    "for suffix in perdelta(date(1958, 8, 9), date(2012, 12, 31), timedelta(days=7)):\n",
    "    prefix = 'http://www.billboard.com/charts/hot-100/'\n",
    "    link = prefix + str(suffix)\n",
    "    urls.append(link)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://www.billboard.com/charts/hot-100/1958-08-09',\n",
       " 'http://www.billboard.com/charts/hot-100/1958-08-16',\n",
       " 'http://www.billboard.com/charts/hot-100/1958-08-23']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gets artist name from page\n",
    "def get_artist(soup):\n",
    "    artist = soup.find_all(class_=\"chart-list-item__artist\")\n",
    "    artists = []\n",
    "    for item in artist:\n",
    "        item = item.text.strip()\n",
    "        artists.append(item)\n",
    "    return artists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gets song title from page\n",
    "def get_track(soup):\n",
    "    track = soup.find_all(class_=\"chart-list-item__title-text\")\n",
    "    tracks = []\n",
    "    for item in track:\n",
    "        item = item.text.strip()\n",
    "        tracks.append(item)\n",
    "    return tracks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get artist and track title from each URL. Append to list of dataframes, merge the dataframes and save to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9min 54s, sys: 7.94 s, total: 10min 2s\n",
      "Wall time: 2h 52min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "frames = []\n",
    "billboard_dict = {}\n",
    "for url in urls:\n",
    "    soup = get_soup(url)\n",
    "    artist = get_artist(soup)\n",
    "    track = get_track(soup)\n",
    "    dates = [parser.parse(url.split('/')[5])]*len(artist)\n",
    "    df = pd.DataFrame({'artist': artist, 'track': track, 'publish_date': dates})\n",
    "    time.sleep(2)\n",
    "    frames.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merge = pd.concat(frames).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/billboard_rawdf.pkl', 'wb') as picklefile:\n",
    "    pickle.dump(df_merge, picklefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/billboard_rawdf.pkl', 'rb') as picklefile:\n",
    "    df_bb = pickle.load(picklefile, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete any duplicated songs (i.e. popular for several weeks)\n",
    "df2 = df_merge.drop_duplicates(['track'], keep='first').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/billboard_unique.pkl', 'wb') as picklefile:\n",
    "    pickle.dump(df2, picklefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleanup of df with the way it was encoded\n",
    "def text_clean(x):\n",
    "    try:\n",
    "        x = x.decode('utf-8')\n",
    "    except:\n",
    "        None\n",
    "    return x\n",
    "\n",
    "df2 = df2.applymap(text_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>track</th>\n",
       "      <th>publish_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Domenico Modugno</td>\n",
       "      <td>Nel Blu Dipinto Di Blu (Volaré)</td>\n",
       "      <td>1958-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Perez Prado And His Orchestra</td>\n",
       "      <td>Patricia</td>\n",
       "      <td>1958-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bobby Darin</td>\n",
       "      <td>Splish Splash</td>\n",
       "      <td>1958-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kalin Twins</td>\n",
       "      <td>When</td>\n",
       "      <td>1958-08-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jack Scott</td>\n",
       "      <td>My True Love</td>\n",
       "      <td>1958-08-09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          artist                            track publish_date\n",
       "0               Domenico Modugno  Nel Blu Dipinto Di Blu (Volaré)   1958-08-09\n",
       "1  Perez Prado And His Orchestra                         Patricia   1958-08-09\n",
       "2                    Bobby Darin                    Splish Splash   1958-08-09\n",
       "3                    Kalin Twins                             When   1958-08-09\n",
       "4                     Jack Scott                     My True Love   1958-08-09"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export to file\n",
    "df2.to_csv(\"./Data/billboard.csv\", encoding=\"utf-8\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21176, 3)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}