import json
import operator
import os
from collections import defaultdict, Counter

import spacy


def are_invalid(lyrics: str):
    placeholders = ["Please check back once the song has been released",
                    "Lyrics for this song have yet",
                    "Tell us that you would like to have the lyrics of this song"]
    for placeholder in placeholders:
        if lyrics.find(placeholder) != -1:
            return True
    return False


def cleanup(lyrics):
    lines = lyrics.split("\n")
    filtered = [l for l in lines if
                l is not None and len(l) and
                l[0] not in ["#", "["] and
                "Paroles de la chanson" not in l and
                "Paroles de Même" not in l]

    return "\n".join(filtered)


def analyse():
    path = "../data/"
    stats = {
        "avg_lines": defaultdict(lambda: 0),
        "avg_newlines": defaultdict(lambda: 0),
        "missing": defaultdict(lambda: 0)
    }

    print("Go")
    files = os.listdir(path)
    print("Listed data files.")
    nlp = spacy.load("fr_core_news_md")
    print("Loaded spaCy.")

    for filename in files:
        with open(path + filename) as f:
            content = json.load(f)
            print("Loaded", filename)
            artist = content["name"]
            songs = content["songs"]

            print("\n## %s ##" % artist)
            keywords = []

            occurences = {}
            for song in songs:
                if song["lyrics"] is not None:
                    keyword = "anar"
                    occurences[song["title"]] = song["lyrics"].lower().count(keyword)

            if any([i > 0 for i in occurences.values()]):
                print("%s chez %s: %i" % (keyword, artist, len(occurences)))
                for pair in sorted(occurences.items(), key=operator.itemgetter(1), reverse=True):
                    print(pair)
            else:
                print("%s ne parle pas de %s" % (artist, keyword))

            for song in songs:
                # title = song["title"]
                if song["lyrics"] is None:
                    stats["missing"][artist] += 1
                else:
                    lyrics: str = song["lyrics"]

                    if are_invalid(lyrics):
                        stats["missing"][artist] += 1
                    else:
                        lyrics = cleanup(lyrics)
                        stats["avg_lines"][artist] += len(lyrics)
                        stats["avg_newlines"][artist] += lyrics.count("\n")

                        doc = nlp(lyrics)
                        keywords.extend([token.text for token in doc if
                                         token.is_stop != True and
                                         token.is_punct != True and
                                         token.pos_ in ["PRON", "NOUN", "ADJ", "VERB", "INTJ", "X"]])

            # five most common noun tokens
            noun_freq = Counter(keywords)
            common_nouns = noun_freq.most_common(20)

            final_missing = stats["missing"][artist]
            final_songs = len(songs) - final_missing

            stats["avg_lines"][artist] /= final_songs
            stats["avg_newlines"][artist] /= final_songs

            print("%s: %i songs, on average %i/%i long (%i missing)." % (
                artist, final_songs,
                stats["avg_lines"][artist],
                stats["avg_newlines"][artist],
                final_missing))
            print(common_nouns)


if __name__ == '__main__':
    analyse()
