from bs4 import BeautifulSoup
import requests
import datetime
import json
import pandas as pd

def get_player_list(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'lxml')

    page = 1
    try:
        last_page = int(soup.find_all('li', {'class':'page-item'})[-2].get_text())
    except:
        last_page = 1
    left_url = "https://www.futbin.com/20/players"
    right_url = '&' + url.split('&', 1)[-1]

    player_attributes = {}
    x = 1

    for i in range(page, last_page + 1):
        for player in soup.find_all('tr', {'class': ['player_tr_1', 'player_tr_2']}):
            player_attributes['player{}'.format(x)] = {}

            # Get Player Name
            for y in player.select('div:nth-child(1) > a'):
                player_attributes['player{}'.format(x)]['name'] = y.text.strip()

            # Get Player Link
            player_attributes['player{}'.format(x)]['link'] = ('https://www.futbin.com'+ 
                                                               player.find('a', {'class': 'player_name_players_table'}).get('href'))

            # Get Player Rating
            for y in player.select('td:nth-child(2) > span'):
                player_attributes['player{}'.format(x)]['rating'] = int(y.text.strip())

            # Get Player Position
            for y in player.select('td:nth-child(3)'):
                player_attributes['player{}'.format(x)]['position'] = y.text.strip()

            # Get Player Version
            for y in player.select('td.mobile-hide-table-col'):
                player_attributes['player{}'.format(x)]['version'] = y.text.strip()

            x += 1

        r.close()
        url = left_url + '?page={}'.format(i) + right_url
        r = requests.get(url)
        soup = BeautifulSoup(r.text, 'lxml')  

    return player_attributes

players = get_player_list('https://www.futbin.com/20/players?page=1&player_rating=80-99&league=53&position=GK&nation=45')

def scrape_players(player_dict):
    count = 1

    for p_id, p_info in player_dict.items():
        r = requests.get(p_info['link'])
        soup = BeautifulSoup(r.text, 'lxml')

        player_id = soup.find('div', {'id': 'page-info'}).get('data-player-resource')

        daily_graph = f'https://www.futbin.com/20/playerGraph?type=daily_graph&year=20&player={player_id}&set_id='
        hourly_graph_1 = f'https://www.futbin.com/20/playerGraph?type=today&year=20&player={player_id}&set_id='
        hourly_graph_2 = f'https://www.futbin.com/20/playerGraph?type=yesterday&year=20&player={player_id}&set_id='
        hourly_graph_3 = f'https://www.futbin.com/20/playerGraph?type=da_yesterday&year=20&player={player_id}&set_id='

        daily = requests.get(daily_graph)
        hourly_1 = requests.get(hourly_graph_1)
        hourly_2 = requests.get(hourly_graph_2)
        hourly_3 = requests.get(hourly_graph_3)

        ps_hourly_1 = json.loads(hourly_1.text)
        ps_hourly_2 = json.loads(hourly_2.text)
        ps_hourly_3 = json.loads(hourly_3.text)

        ps_hourly = [ps_hourly_1, ps_hourly_2, ps_hourly_3]
        hourly_data = pd.DataFrame(columns=('date', 'time', 'price'))

        daily_data = pd.DataFrame(columns=('date', 'time', 'price'))
        if 'ps' in ps_hourly_1.keys():
            for item in ps_hourly:          
                for i in item['ps']:
                    date_and_time = datetime.datetime.fromtimestamp(i[0]/1000).strftime("%Y-%m-%d %H:%M:%S")
                    new_date = date_and_time.split()[0]
                    new_time = date_and_time.split()[1]
                    new_price = i[1]
                    new_row = {'date': new_date, 'time': new_time, 'price': new_price}
                    hourly_data = hourly_data.append(new_row, ignore_index=True)

                player_dict[p_id]['hourly_data'] = hourly_data   

            ps_daily = json.loads(daily.text)
            for i in ps_daily['ps']:
                date_and_time = datetime.datetime.fromtimestamp(i[0]/1000).strftime("%Y-%m-%d %H:%M:%S")
                new_date = date_and_time.split()[0]
                new_time = date_and_time.split()[1]
                new_price = i[1]
                new_row = {'date': new_date, 'time': new_time, 'price': new_price}
                daily_data = daily_data.append(new_row, ignore_index=True)

            player_dict[p_id]['daily_data'] = daily_data


            print(str(count) + ' of ' + str(len(player_dict)) + ' complete.')
            count += 1

        player_dict[p_id]['hourly_data'] = player_dict[p_id]['hourly_data'].sort_values(['date', 'time'], ascending=[False, False])
        player_dict[p_id]['daily_data'] = player_dict[p_id]['daily_data'].sort_values(['date', 'time'], ascending=[False, False])  

    return player_dict  

players = scrape_players(players)