# Import packages
import requests
from bs4 import BeautifulSoup

# Request header
headers = {"User-Agent": "Chrome/68.0.3440.106"}

# Request parameters
parameters = {
    'find_desc': 'Restaurants',
    'find_loc':'San Francisco CA',
    'start':'0'}

# Yelp search url
url='https://www.yelp.com/search'

# Get the number of pages return by request (Ex: 'Page 1 of 34') 
# Each page shows 30 items
response = requests.get(url, params=parameters)
# Create BeautifulSoup Object
soup = BeautifulSoup(response.text, 'html.parser')
# Extract the number of pages
string_pages = soup.findAll('div', attrs={'class':'page-of-pages arrange_unit arrange_unit--fill'})[0].text
number_of_pages = int(string_pages.split('of ')[1])

# Variable that stores the data
data = []

# Start page
pagination = 0

# For each pagination extract the items
for i in range(number_of_pages):

    # Produces the request and saves the response
    response=requests.get(url, params=parameters)
    # Create a Soup object
    soup = BeautifulSoup(response.text, 'html.parser')

    # count 30 pages
    #count = 1

    print('Extract data ... page {} from {}'.format(int(pagination/30)+1, number_of_pages))

    # Extract the info
    for a in soup.findAll('div', attrs={'class':'media-story'}):

        # Validate if the info exists
        try:
            address = a.find('address').text.strip()
        except:
            address = ''

        try:
            name = a.find('a', attrs={'class':'biz-name js-analytics-click'}).text
        except:
            name = ''

        try:
            reviews = a.find('span', attrs={'class':'review-count rating-qualifier'}).text.strip().split()[0]
        except:
            reviews = ''

        try:
            phone = a.find('span', attrs={'class':'biz-phone'}).text.strip()
        except:
            phone = ''

        try: 
            neighborhood = a.find('span', attrs={'class':'neighborhood-str-list'}).text.strip()
        except:
            neighborhood = ''

        # Saves info in a dictionary
        info = {
        'name' : name,
        'reviews' : reviews,
        'phone' : phone,
        'address' : address,
        'neighborhood' : neighborhood
        }
        # saves dictionary in the data variable
        data.append(info)

        # Each page shows 30 items, but the Soup Object find more (garbage)
        # To avoid some erros in processing, This code guarantees that only 30 items will be scraped
        #count += 1
        #if count == 30: break

    # Increment the start item ID (pagination)
    pagination+=30
    parameters['start'] = pagination

data[:5]

from pandas import DataFrame

restaurants = DataFrame(data)

# Show the ten firsts items
restaurants.head(10)

# Check if there is any null value in DataFrame
restaurants.isnull().any().any()

# Check if there is any empty value in DataFrame
no_address = len(restaurants[restaurants['address'] == ''])
no_name = len(restaurants[restaurants['name'] == ''])
no_neighborhood = len(restaurants[restaurants['neighborhood'] == ''])
no_phone = len(restaurants[restaurants['phone'] == ''])
no_reviews = len(restaurants[restaurants['reviews'] == ''])

print('Column \t\t empty values')
print('address \t ', no_address)
print('name \t\t ', no_name)
print('neighborhood \t ', no_neighborhood)
print('phone \t\t ', no_phone)
print('reviews \t ', no_reviews)

restaurants = restaurants[restaurants['address'] != '']
restaurants = restaurants[restaurants['name'] != '']
restaurants = restaurants[restaurants['neighborhood'] != '']
restaurants = restaurants[restaurants['phone'] != '']
restaurants = restaurants[restaurants['reviews'] != '']

# Check if there is any duplicate value in DataFrame
restaurants[restaurants.duplicated()]

restaurants.drop_duplicates(inplace=True)

restaurants = restaurants[['name', 'neighborhood', 'address', 'phone', 'reviews']]
restaurants.head(5)

restaurants.dtypes

restaurants['reviews'] = restaurants['reviews'].astype(int)
restaurants.dtypes

restaurants.to_csv('datasets/yelp_restaurants_sanFranscisco.csv', index=False)

import matplotlib.pyplot as plt
import numpy as np
% matplotlib inline

# Get top 10 restaurants
top10 = restaurants[['name', 'reviews']].sort_values('reviews',ascending=False).head(10).sort_values('reviews')

# Plot 
top10.plot.barh(legend=False)
plt.yticks(np.arange(10), top10.name.values)
plt.ylabel('RESTAURANT')
plt.xlabel('REVIEWS')
plt.show()

top_neighbor = restaurants['neighborhood'].value_counts().head(10)
top_neighbor.plot.barh(figsize=(7,5))
plt.xlabel('NUMBER OF RESTAURANTS')
plt.ylabel('NEIGHBORHOOD')
plt.show()

(restaurants['neighborhood'].value_counts(normalize=True).head(10)*100).plot.barh(figsize=(7,5))
plt.xlabel('PROPORTION (%)')
plt.ylabel('NEIGHBORHOOD')
plt.show()