# Here's an outline of how this program works.
# Firstly, we load a dictionary mapping GSIS identifier to a dictionary of
# player meta data. This comes from either the flag `json-update-file` or
# nflgame's "players.json" file. We then build a reverse map from profile
# identifier (included in player meta data) to GSIS identifier.
#
# We then look at all players who have participated in the last week of
# play. Any player in this set that is not in the aforementioned mapping
# has his GSIS identifier and name (e.g., `T.Brady`) added to a list of
# players to update.
#
# (N.B. When the initial mappings are empty, then every player who recorded
# a statistic since 2009 is added to this list.)
#
# For each player in the list to update, we need to obtain the profile
# identifier. This is done by sending a single HEAD request to the
# `gsis_profile` URL. The URL is a redirect to their canonical profile page,
# with which we extract the profile id. We add this mapping to both of the
# mappings discussed previously. (But note that the meta data in the GSIS
# identifier mapping is incomplete.)
#
# We now fetch the roster lists for each of the 32 teams from NFL.com.
# The roster list contains all relevant meta data *except* the GSIS identifier.
# However, since we have a profile identifier for each player (which is
# included in the roster list), we can connect player meta data with a
# particular GSIS identifier. If we happen to see a player on the roster that
# isn't in the mapping from profile identifier to GSIS identifier, then we need
# to do a full GET request on that player's profile to retrieve the GSIS
# identifier. (This occurs when a player has been added to a roster but hasn't
# recorded any statistics. e.g., Rookies, benchwarmers or offensive linemen.)
#
# We overwrite the initial dictionary of player meta data for each player in
# the roster data, including adding new entries for new players. We then save
# the updated mapping from GSIS identifier to player meta data to disk as JSON.
# (The JSON dump is sorted by key so that diffs are meaningful.)
#
# This approach requires a few thousand HEAD requests to NFL.com on the first
# run. But after that, most runs will only require 32 requests for the roster
# list (small potatoes) and perhaps a few HEAD/GET requests if there happens to
# be a new player found.

from __future__ import absolute_import, division, print_function
import argparse
import json
import multiprocessing.pool
import os
import re
import sys
import traceback

import httplib2

from bs4 import BeautifulSoup

try:
    import lxml.html  # noqa
    PARSER = 'lxml'
except ImportError:
    try:
        import html5lib  # noqa
        PARSER = 'html5lib'
    except ImportError:
        PARSER = 'html.parser'

import nflgame
import nflgame.live
import nflgame.player

urls = {
    'roster': 'http://www.nfl.com/teams/roster?team=%s',
    'gsis_profile': 'http://www.nfl.com/players/profile?id=%s',
}


def new_http():
    http = httplib2.Http(timeout=10)
    http.follow_redirects = True
    return http


def initial_mappings(conf):
    metas, reverse = {}, {}
    try:
        with open(conf.json_update_file) as fp:
            metas = json.load(fp)
        for gsis_id, meta in metas.items():
            reverse[meta['profile_id']] = gsis_id
    except IOError as e:
        eprint('Could not open "%s": %s' % (conf.json_update_file, e))
    # Delete some keys in every entry. We do this to stay fresh.
    # e.g., any player with "team" set should be actively on a roster.
    for k in metas:
        metas[k].pop('team', None)
        metas[k].pop('status', None)
        metas[k].pop('position', None)
    return metas, reverse


def profile_id_from_url(url):
    if url is None:
        return None
    m = re.search('/([0-9]+)/', url)
    return None if m is None else int(m.group(1))


def profile_url(gsis_id):
    resp, content = new_http().request(urls['gsis_profile'] % gsis_id, 'HEAD')
    if resp['status'] != '301':
        return None
    loc = resp['location']
    if not loc.startswith('http://'):
        loc = 'http://www.nfl.com' + loc
    return loc


def gsis_id(profile_url):
    resp, content = new_http().request(profile_url, 'GET')
    if resp['status'] != '200':
        return None
    m = re.search('GSIS\s+ID:\s+([0-9-]+)', content)
    if m is None:
        return None
    gid = m.group(1).strip()
    if len(gid) != 10:  # Can't be valid...
        return None
    return gid


def roster_soup(team):
    resp, content = new_http().request(urls['roster'] % team, 'GET')
    if resp['status'] != '200':
        return None
    return BeautifulSoup(content, PARSER)


def try_int(s):
    try:
        return int(s)
    except ValueError:
        return 0


def first_int(s):
    m = re.search('[0-9]+', s)
    if m is None:
        return 0
    return int(m.group(0))


def first_word(s):
    m = re.match('\S+', s)
    if m is None:
        return ''
    return m.group(0)


def height_as_inches(txt):
    # Defaults to 0 if `txt` isn't parseable.
    feet, inches = 0, 0
    pieces = re.findall('[0-9]+', txt)
    if len(pieces) >= 1:
        feet = try_int(pieces[0])
        if len(pieces) >= 2:
            inches = try_int(pieces[1])
    return feet * 12 + inches


def meta_from_soup_row(team, soup_row):
    tds, data = [], []
    for td in soup_row.find_all('td'):
        tds.append(td)
        data.append(td.get_text().strip())
    profile_url = 'http://www.nfl.com%s' % tds[1].a['href']

    name = tds[1].a.get_text().strip()
    if ',' not in name:
        last_name, first_name = name, ''
    else:
        last_name, first_name = map(lambda s: s.strip(), name.split(','))[:2]

    return {
        'team': team,
        'profile_id': profile_id_from_url(profile_url),
        'profile_url': profile_url,
        'number': try_int(data[0]),
        'first_name': first_name,
        'last_name': last_name,
        'full_name': '%s %s' % (first_name, last_name),
        'position': data[2],
        'status': data[3],
        'height': height_as_inches(data[4]),
        'weight': first_int(data[5]),
        'birthdate': data[6],
        'years_pro': try_int(data[7]),
        'college': data[8],
    }


def meta_from_profile_html(html):
    if not html:
        return html
    try:
        soup = BeautifulSoup(html, PARSER)
        pinfo = soup.find(id='player-bio').find(class_='player-info')

        # Get the full name and split it into first and last.
        # Assume that if there are no spaces, then the name is the last name.
        # Otherwise, all words except the last make up the first name.
        # Is that right?
        name = pinfo.find(class_='player-name').get_text().strip()
        name_pieces = name.split(' ')
        if len(name_pieces) == 1:
            first, last = '', name
        else:
            first, last = ' '.join(name_pieces[0:-1]), name_pieces[-1]
        meta = {
            'first_name': first,
            'last_name': last,
            'full_name': name,
        }

        # The position is only in the