update_bgg_preview

#!/usr/bin/tclsh

set version 2017.01  ;# Spiel17 condensed version
set version 2017.02  ;# Bugfix: ignore random game link on bgg site
set version 2017.03  ;# Bugfix: ignore ° (thanks to Knut)
set version 2017.04  ;# Bugfix: ignore all games before "geekpreview-header-cta"
set version 2017.10  ;# add field for rating (not extracted from bgg yet)
set version 2018.01  ;# extracted booth for Gencon 2018
set version 2018.02  ;# renamed temporary html file, added more unicode chars
set version 2018.03  ;# renamed more publishers and games without unicode
set version 2018.04  ;# support Unicode characters - finally!
set version 2019.01  ;# support new html format for Gencon 2019
set version 2019.02  ;# bugfix for new html format

# search "old" csv file (as base) and new html files
set csvfile ""
set htmlfile ""
set showhelp 0
set debug 0
foreach argument $argv {
    if { [regexp {.csv$} $argument] } {
        if { [string compare $csvfile ""] == 0 } {
            set csvfile $argument
        } else {
            puts "Error: Second csv file given!"
            set showhelp 1
        }
    } elseif { [regexp {.html$} $argument] } {
        if { [string compare $htmlfile ""] == 0 } {
            set htmlfile $argument
        } else {
            puts "Error: Second html file given!"
            set showhelp 1
        }
    } elseif { [string compare $argument "--help"] == 0 } {
        set showhelp 1
    } elseif { [string compare $argument "--debug"] == 0 } {
        set debug 1
    } elseif { [string compare $argument ""] != 0 } {
        puts "Error: Unknown file: $argument"
        set showhelp 1
    }
}

if { !$showhelp && [string compare $htmlfile ""] == 0 } {
    puts "Error: No valid html file given as argument!"
    set showhelp 1
}

proc putsDebug { line } {
    global debug
    if { $debug } {
        puts "<DBG> $line"
    }
}

putsDebug "Args: $argc $argv"
putsDebug "HTML: $htmlfile"
putsDebug "CSV: $csvfile"

if { $showhelp } {
    puts "Expected: [file nativename $argv0] \[--debug\] \[datei.csv\] codensed_view.html \[> datei.csv\]"
    puts "For help: [file nativename $argv0] --help"
    exit 1
}

# -----------------------------------------------

# convert each html file so that it can be parsed easier
set date [clock format [clock seconds] -format "%y%m%d"]
regsub {\.html$} $htmlfile "_$date.html" newfilename
#puts "Read HTML:  $htmlfile / Write HTML: $newfilename"
set infile [open $htmlfile r]
set outfile [open $newfilename w]
while { [gets $infile line] >= 0 } {
    # Remove tabs and white spaces around links
    regsub -all {(<a[^>]*>)\s*([^\t]*)\s*(</a>)} $line {\1\2\3} line

    # Remove tabs and white spaces around price tag
    regsub -all {(<strong ng-if="[^"]*itemctrl.item.showprice[^>]*>)\s*([^\t]*)\s*(</strong>)} $line {\1\2\3} line ;#"

    # Remove tabs and white spaces around availability tag
    regsub -all {(<span ng-if="itemctrl.item.pretty_availability_status"[^>]*>)\s*([^\t]*)\s*(</span>)} $line {\1\2\3} line

    # Remove tabs and white spaces around priority tag
    regsub -all {(<button ng-if="ctrl.userinfo.priority[^>]*">)\s*([^\t]*)\s*(<)} $line {\1\2\3} line

    # Tabs are used mostly as separator between entries
    regsub -all "\t" $line "\n" line

    # Comment lines are used as separators as well
    regsub -all "<!---->" $line "\n" line

    # regsub -all {(<div class="col-xs-2">)\s*([^\t]*)\s*(</div>)} $line {\1\2\3} line

    # Remove all unnecessary white spaces
    #regsub -all {>\s+} $line {>} line
    #regsub -all {\s+<} $line {<} line

    puts $outfile $line
}
close $outfile
close $infile
set htmlfile $newfilename


# -----------------------------------------------

proc readCsvEntry { line } {
    global g_csvVersion
    global g_boothList
    global g_publisherList
    global g_gameList
    global g_ratingList
    global g_playersList
    global g_durationList
    global g_languageList
    global g_priceList
    global g_prioList
    global g_remarkList
    global g_bggLinkList
    global g_inCsvOnly

    if { ( $g_csvVersion >= 2019.01 && $g_csvVersion < 2019.10 ) } {
        # v2019.0X
        # puts "Booth,Publisher,Game,Rating,Players,Dur,Lang,Price,Prio,Remarks,BGG"
        if { [regexp {^([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)$} $line all booth publisher game rating players duration language price prio remarks bgglink ] } {
            set publisher [string trim $publisher {"}] ;#"
            set game [string trim $game {"}] ;#"
            set entry [createEntry $publisher $game]
            set g_boothList($entry)     [string trim $booth {"}] ;#"
            set g_publisherList($entry) [string trim $publisher {"}] ;#"
            set g_gameList($entry)      [string trim $game {"}] ;#"
            set g_ratingList($entry)    [string trim $rating {"}] ;#"
            set g_playersList($entry)   [string trim $players {"}] ;#"
            set g_durationList($entry)  [string trim $duration {"}] ;#"
            set g_languageList($entry)  [split [string trim $language {"}] {/}] ;#"
            set g_priceList($entry)     [string trim $price {"}] ;#"
            set g_prioList($entry)      [string trim $prio {"}] ;#"
            set g_remarkList($entry)    [string trim $remarks {"}] ;#"
            set g_bggLinkList($entry)   [string trim $bgglink {"}] ;#"
            set g_inCsvOnly($entry)     1
            if { [string first $g_remarkList($entry) "BGG"] >= 0 } {
                # entries with BGG as comment are manually added
                set g_inCsvOnly($entry)     0
            }
        } else {
            error "Wrong CSV line: $line"
        }
    } else {
        error "readCsvEntry Unknown version: $g_csvVersion"
    }
}

proc checkCsvVersion { file } {
    global g_csvVersion
    gets $file line
    regexp {^"?v([^",]*)} $line all g_csvVersion
    # puts "Version: $g_csvVersion"
    gets $file line
    if { ( $g_csvVersion >= 2019.01 && $g_csvVersion < 2019.10 ) } {
        # v2019.0X
        # puts "Booth,Publisher,Game,Rating,Players,Dur,Lang,Price,Prio,Remarks,BGG"
        regsub -all {"} $line {} line  ;#"
        if { [string compare $line {Booth,Publisher,Game,Rating,Players,Dur,Lang,Price,Prio,Remarks,BGG}] != 0 } {
            error "CSV-format does not match version $g_csvVersion: $line"
        }
    } else {
        error "checkCsvVersion Unknown version: $g_csvVersion"
    }
}

proc readCsv { filename } {
    global g_csvVersion
    set file [open $filename r]
    checkCsvVersion $file
    while { [gets $file line] >= 0 } {
        readCsvEntry $line
    }
}

# -----------------------------------------------

proc convertUnicode { line } {
    regsub -all {,} $line {} line
    regsub -all {&amp;} $line {\&} line
    regsub -all {&nbsp;} $line { } line
    regsub -all {−} $line {-} line
    regsub -all {–} $line {-} line
    regsub -all {&minus;} $line {-} line
    regsub -all {&#039;} $line {} line
    regsub -all {´} $line {'} line
    regsub -all {’} $line {'} line
    regsub -all {\t+} $line {} line
    regsub -all {­} $line {} line
    return $line
}

proc createEntry { publisher game } {
    set entry "$publisher,$game"
    # regsub -all {,} $entry {} entry
    regsub -all {\.} $entry {} entry
    regsub -all {!} $entry {} entry
    regsub -all {\?} $entry {} entry
    regsub -all {:} $entry {} entry
    regsub -all {'} $entry {} entry
    regsub -all {\-} $entry {} entry
    regsub -all { } $entry {} entry
    regsub -all {\(} $entry {} entry
    regsub -all {\)} $entry {} entry
    regsub -all {/} $entry {} entry
    regsub -all {­­­} $entry {} entry
    regsub -all {\+} $entry {} entry
    regsub -all {\+} $entry {} entry
    regsub {^[0-9]*} $entry {} entry
    return $entry
}

proc storeAsNewGame { entry } {
    global g_boothList
    global g_publisherList
    global g_gameList
    global g_ratingList
    global g_playersList
    global g_durationList
    global g_languageList
    global g_priceList
    global g_prioList
    global g_remarkList
    global g_bggLinkList
    global g_inCsvOnly

    global g_publisher
    global g_booth
    global g_game
    global g_players
    global g_playtime
    global g_price
    global g_prio
    global g_bgglink
    global g_language

    set g_publisherList($entry) $g_publisher
    set g_boothList($entry) $g_booth
    set g_gameList($entry) $g_game
    set g_ratingList($entry) {}
    set g_playersList($entry) $g_players
    set g_durationList($entry) $g_playtime
    set g_languageList($entry) $g_language
    set g_priceList($entry) $g_price
    set g_prioList($entry) $g_prio
    set g_remarkList($entry) {}
    set g_bggLinkList($entry) $g_bgglink
    set g_inCsvOnly($entry) 0

    unset g_game
    unset g_players
    unset g_playtime
    unset g_language
    unset g_price
    unset g_prio
    unset g_bgglink
}

proc storeNumPlayers { entry } {
    global g_playersList
    global g_players
    if { [string length $g_players] > 0 } {
        if { [string compare $g_playersList($entry) $g_players] != 0 } {
            if { [string length $g_playersList($entry)] == 0 } {
                set g_playersList($entry) $g_players
            } else {
                # puts "> $g_playersList($entry) -> $g_players $entry"
                set g_playersList($entry) $g_players
            }
        } ;# else identical
    } ;# else nothing set
    unset g_players
}

proc storePlaytime { entry } {
    global g_durationList
    global g_playtime
    if { [string length $g_playtime] > 0 } {
        if { [string compare $g_durationList($entry) $g_playtime] != 0 } {
            if { [string length $g_durationList($entry)] == 0 } {
                set g_playersList($entry) $g_playtime
            } else {
                # puts "> $g_durationList($entry) -> $g_playtime $entry"
                set g_durationList($entry) $g_playtime
            }
        } ;# else identical
    } ;# else nothing set
    unset g_playtime
}

proc storeLanguage { entry } {
    global g_languageList
    global g_language
    if { [llength $g_language] > 0 } {
        # we only add new languages
        foreach lang $g_language {
            if { [lsearch $lang $g_languageList($entry)] == -1 } {
                lappend $lang $g_languageList($entry)
            } ;# else language already stored
        }
    } ;# else nothing set

    # Language from spielbox depends on publisher
    unset g_language
}

proc storePrice { entry } {
    global g_priceList
    global g_price
    if { [string length $g_price] > 0 } {
        if { [string compare $g_priceList($entry) $g_price] != 0 } {
            if { [string length $g_priceList($entry)] == 0 } {
                set g_priceList($entry) $g_price
            } else {
                # puts "> $g_priceList($entry) -> $g_price $entry"
                set g_priceList($entry) $g_price
            }
        } ;# else identical
    } ;# else nothing set
    unset g_price
}

proc storeBooth { entry } {
    global g_boothList
    global g_booth
    if { [info exists g_booth] && [string length $g_booth] > 0 } {
        if { [string compare $g_boothList($entry) $g_booth] != 0 } {
            if { [string length $g_boothList($entry)] == 0 } {
                set g_boothList($entry) $g_booth
            } else {
                # puts "> $g_boothList($entry) -> $g_booth $entry"
                set g_boothList($entry) $g_booth
            }
        } ;# else identical
    } ;# else nothing set
    # do not unset booth because it is not repeated
}

proc storeBggLink { entry } {
    global g_bggLinkList
    global g_bgglink
    if { [string length $g_bgglink] > 0 } {
        set g_bggLinkList($entry) $g_bgglink
    } ;# else nothing set
    unset g_bgglink
}

proc storePrio { entry } {
    global g_prioList
    global g_prio
    if { [string length $g_prio] > 0 } {
        if { [string compare $g_prioList($entry) $g_prio] != 0 } {
            if { [string length $g_prioList($entry)] == 0 } {
                set g_prioList($entry) $g_prio
            } else {
                # Wir ueberschreiben den Status nur, wenn nicht eine Priozahl
                # manuell eingegeben wurde.
                if { ![string is integer $g_prioList($entry)] } {
                    # puts "> $g_prioList($entry) -> $g_prio $entry"
                    set g_prioList($entry) $g_prio
                }
            }
        } ;# else identical
    } ;# else nothing set
    unset g_prio
}

proc storeAsSingleGame { entry } {
    global g_inCsvOnly
    set g_inCsvOnly($entry)      0

    # Publisher nicht loeschen, da der nicht wiederholt wird!
    global g_publisherList
    global g_publisher
    set g_publisherList($entry) $g_publisher

    global g_gameList
    global g_game
    set g_gameList($entry) $g_game
    unset g_game

    storeNumPlayers $entry
    storePlaytime $entry
    storeLanguage $entry
    storePrice $entry
    storeBooth $entry
    storePrio $entry
    storeBggLink $entry
}

proc storeGame { } {
    global g_gameList
    global g_publisher
    global g_game
    global g_counter
    global g_remarkList

    if { [info exists g_publisher] && [info exists g_game] } {
        set entry [createEntry $g_publisher $g_game]
        incr g_counter
        if { ![info exists g_gameList($entry)] } {
            storeAsNewGame $entry
        } else {
            storeAsSingleGame $entry
        }
    } elseif { [info exists g_game] } {
        error "Game exists, but not publisher: $g_game"
    } elseif { [info exists g_publisher] } {
        # No error, it's normal that at the end of the list the publisher
        # is still set because it is not reset
        # error "Publisher exists, but not game: $g_publisher"
        unset g_publisher
    }
}

proc checkPublisherBGG { line } {
    global g_publisher
    global g_booth
    global g_publisherJustSet

    if { [regexp {<a ng-href="/boardgamepublisher/[^"]*"[^>]*>\s*([^<]*)\s*</a>} $line all publisher] } { ;#"
        set publisher [string trim $publisher]
        set publisher [convertUnicode $publisher]
        
        # Save previous stored game!
        storeGame
        
        set g_publisher $publisher
        set g_publisherJustSet 1
        set g_booth {}
        putsDebug "Publisher: $g_publisher"
    }
    if { [info exists g_publisher] &&
         [string compare $g_publisher "Self-Published"] == 0 } {
        # Clear Booth number if self published because it should be listed under
        # the game not the "publisher".
        set g_booth {}
    }
}

proc checkBoothBGG { line } {
    global g_booth
    if { [regexp {<span ng-if="parentitem.location"[^>]*>Location: Hall ([0-9]+): ([A-Z0-9]+)</span>} $line all hall number] ||
    [regexp {<span ng-if="parentitem.location"[^>]*>Location: ([^<]*)</span>} $line all number] } {
        if { [info exists hall] } {
            set booth [string trim "$hall-$number"]
        } else {
            regsub -all {, *} $number {/} number
            regsub -all {—} $number {-} number
            regsub -all {&amp;} $number {/} number
            set booth [string trim $number]
        }
        if { [string compare $g_booth ""] == 0 } {
            set g_booth $booth
            putsDebug "Booth: $g_booth"
        } elseif { [string compare $g_booth $booth] != 0 } {
            # Already a booth set. This should not happen.
            error "Second booth set: $g_booth -> $booth"
        }
    }
}

proc checkGameTitleBGG { line } {
    global g_game
    global g_bgglink
    global g_price
    global g_prio
    global g_players
    global g_playtime
    global g_language
    global g_publisherJustSet

    if { ( [regexp {<a ng-href="(/boardgame/[^"]*)"[^>]*>\s*([^<]*)\s*</a>} $line all link game] ||
           [regexp {<a ng-href="(/boardgameexpansion/[^"]*)"[^>]*>\s*([^<]*)\s*</a>} $line all link game] ) } {
        set game [string trim $game]
        set game [convertUnicode $game]

        # Save previous stored game!
        if { !$g_publisherJustSet } {
            storeGame
        }
        set g_publisherJustSet 0

        set g_game $game
        if { [string first "https://boardgamegeek.com" $link] == 0 } {
            set g_bgglink "$link"
        } else {
            set g_bgglink "https://boardgamegeek.com$link"
        }
        putsDebug "Game: $g_game"
        
        set g_players {}
        set g_playtime {}
        set g_language {}
        set g_price {}
        set g_prio {}
    }
}

proc checkPriceBGG { line } {
    global g_price

    if { [regexp {<span ng-if="itemctrl.item.pretty_availability_status[^>]*">([^<]*)</span>} $line all price] } {
        set g_price [string trim $price]
        putsDebug "Avail: $g_price"
    }
    if { [regexp {<strong ng-if=.*itemctrl.item.showprice[^>]*>([^<]*)</strong>} $line all price] } {
        set g_price [string trim $price]
        set g_price [convertUnicode $g_price]
        putsDebug "Price: $g_price"
    }
}

proc checkPriorityBGG { line } {
    global g_prio
    if { [regexp {<button ng-if="ctrl.userinfo.priority[^>]*">([^<]*)<} $line all prio] } {
        set prio [string trim $prio]
        if { [string compare "Remove Priority" $prio] != 0 } {
            set g_prio $prio
            putsDebug "Prio: $g_prio"
        }
    }
}

proc readHtmlBGG { filename } {
    # Syntax 2019:
    # <a.*href="https://boardgamegeek.com/boardgamepublisher/.*">XXXXXX</a> 
    # <span ng-if="parentitem.location">Location: XXXXXX</span>
    # <a.*href="https://boardgamegeek.com/boardgame/.*">XXXXXX</a>
    # <span ng-if="item.pretty_availability_status">XXXXXX</span>
    # <strong ng-if=".*itemctrl.item.showprice*">XXXXXX</strong>
    # <button ng-if="ctrl.userinfo.priority.*>XXXXXX<.*</button>

    set gameListStarted 0
    set file [open $filename r]
    while { [gets $file line] >= 0 } {
        # putsDebug $line
        if { [regexp {<div class="geekpreview-header-cta.*">} $line] } {
            set gameListStarted 1
        }
        if { $gameListStarted } {
            checkPublisherBGG $line
            checkBoothBGG $line
            checkGameTitleBGG $line
            checkPriceBGG $line
            checkPriorityBGG $line
        }
    }
    close $file
}

# -----------------------------------------------

proc printEntries { } {
    global g_boothList
    global g_publisherList
    global g_gameList
    global g_ratingList
    global g_playersList
    global g_durationList
    global g_languageList
    global g_priceList
    global g_prioList
    global g_remarkList
    global g_bggLinkList
    global g_inCsvOnly
    global version

    puts "v$version,,,,,,,,,,,,,"
    puts "Booth,Publisher,Game,Rating,Players,Dur,Lang,Price,Prio,Remarks,BGG"

    foreach entry [lsort -nocase -ascii [array names g_gameList]] {
        puts -nonewline $g_boothList($entry)
        puts -nonewline ","
        puts -nonewline $g_publisherList($entry)
        puts -nonewline ","
        puts -nonewline $g_gameList($entry)
        puts -nonewline ","
        puts -nonewline $g_ratingList($entry)
        puts -nonewline ","
        puts -nonewline $g_playersList($entry)
        puts -nonewline ","
        puts -nonewline $g_durationList($entry)
        puts -nonewline ","
        puts -nonewline [join [lsort $g_languageList($entry)] {/}]
        puts -nonewline ","
        puts -nonewline $g_priceList($entry)
        puts -nonewline ","
        if { $g_inCsvOnly($entry) } {
            # entry is only in csv and not in bgg list
            # mark it as deleted
            puts -nonewline "Deleted: $g_prioList($entry)"
        } else {
            puts -nonewline $g_prioList($entry)
        }
        puts -nonewline ","
        puts -nonewline $g_remarkList($entry)
        puts -nonewline ","
        puts -nonewline $g_bggLinkList($entry)
        puts ""
    }
}

# -----------------------------------------------

# read "old" csv data (as base)
if { [string compare $csvfile ""] } {
    readCsv $csvfile
}

# add new information from html file
readHtmlBGG $htmlfile
# Save last stored game!
storeGame

# print all entries as csv
if { !$debug } {
    printEntries
}