Kronos / (OLD)User Glossary for MTL

// ==UserScript==
// @name         (OLD)User Glossary for MTL
// @version      1.3.5
// @license      MIT
// @namespace    lnmtl_glossary
// @description  User glossary fetched from google docs spreadsheet for lnmtl.com
// @author       Kronos
// @match        http*://lnmtl.com/*
// @require      https://code.jquery.com/jquery-2.2.4.min.js
// @require      https://cdnjs.cloudflare.com/ajax/libs/tabletop.js/1.5.2/tabletop.min.js
// @require      https://userscripts-mirror.org/scripts/source/107941.user.js
// @grant        GM_setValue
// @grant        GM_getValue
// ==/UserScript==

const storageCleanupInterval = 3600000 * 30;
const glossaryChunkSize = 256;
const novelRefreshTime = 6;
let publicSpreadsheetUrl = GM_SuperValue.get("publicSpreadsheetUrl", "");
let replaceInOriginal = GM_SuperValue.get("repalceInOriginal", false);
let noWordWrap = GM_SuperValue.get("noWordWrap", false);
let reduceContrast = GM_SuperValue.get("reduceContrast", false);
let useLNMTLGlossary = GM_SuperValue.get("useLNMTLGlossary", false);
let applyLNMTLGlossary = GM_SuperValue.get("applyLNMTLGlossary", false);
let autoSolveConflicts = GM_SuperValue.get("autoSolveConflicts", false);
let nextStorageCleanup = GM_SuperValue.get("nextStorageCleanup", 0);
let disableGlossaryAnalyzer = GM_SuperValue.get("disableGlossaryAnalyzer", false);
let rawsReplaced = false;
let reload = false;
let LNMTLGlossaryLock = Promise.resolve();
const regexGlossaryTerms = /(\d+)\) (?:(\S+) )?=>(?: (.+))?$/;

let myTime = new Date().getTime();
if (myTime >= nextStorageCleanup || nextStorageCleanup - myTime >= storageCleanupInterval) {
    setTimeout(function() {
        nextStorageCleanup = myTime;
        Object.keys(localStorage).forEach(key => {
            if (key.startsWith("ndata")) {
                let data = JSON.parse(localStorage[key]);
                if (myTime - data.time >= storageCleanupInterval) {
                    localStorage.removeItem("gdata" + data.novel_id);
                    localStorage.removeItem("gvalid" + data.novel_id);
                    sessionStorage.removeItem("gmdata" + data.novel_id);
                    localStorage.removeItem(key);
                }
                else if (nextStorageCleanup > data.time) { nextStorageCleanup = data.time; }
            }});
        GM_SuperValue.set("nextStorageCleanup", nextStorageCleanup += storageCleanupInterval);
    }, 30000);
}


function store(name, value){
    if (typeof(Storage) !== "undefined") {
        try {
            return localStorage.setItem(name, JSON.stringify(value));
        } catch(e) {
            // Cleanup on all errors (full: e.code == 22)
            Object.keys(localStorage).forEach(key => { if (key.startsWith("ndata") || key.startsWith("gdata") || key.startsWith("gvalid")){ localStorage.removeItem(key);}});
            Object.keys(sessionStorage).forEach(key => { if (key.startsWith("gmdata")) { sessionStorage.removeItem(key);}});
        }
    } else {
        GM_SuperValue.set(name, value);
    }
}

function storeSession(name, value){
    if (typeof(Storage) !== "undefined") {
        try {
            return sessionStorage.setItem(name, JSON.stringify(value));
        } catch(e) {
            // Cleanup on all errors (full: e.code == 22)
            Object.keys(sessionStorage).forEach(key => { if (key.startsWith("gmdata")) { sessionStorage.removeItem(key);}});
        }
    }
}

function restoreSession(name, defaultValue){
    if (typeof(Storage) !== "undefined") {
        let item = sessionStorage.getItem(name);
        if(item === null && defaultValue !== undefined)
            return defaultValue;
        return JSON.parse(item);
    }
    return defaultValue;
}

function removeSession(name){
    if (typeof(Storage) !== "undefined") {
        return sessionStorage.removeItem(name);
    }
}

function restore(name, defaultValue){
    if (typeof(Storage) !== "undefined") {
        let item = localStorage.getItem(name);
        if(item === null && defaultValue !== undefined)
            return defaultValue;
        return JSON.parse(item);
    } else {
        // No Web Storage support..
        GM_SuperValue.get(name, defaultValue);
    }
}

function removeStorage(name){
    if (typeof(Storage) !== "undefined") {
        return localStorage.removeItem(name);
    }
}

String.prototype.count = function (regexp) {
    if (typeof regexp === 'string' || regexp instanceof String)
        regexp = new RegExp(regexp.escapeRegExp(), 'g');
    let res = this.match(regexp);
    return res !== null ? res.length: 0;
};

function capitalize(string) {
    return string.charAt(0).toUpperCase() + string.slice(1);
}

function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}

function lock(func) {
    return new Promise(resolve => { func().then(() => resolve()); });
}

function fetchLNMTLNovelData(url, callback) {
    return $.get(url, function(response) { return callback(response); });
}

function loadLNMTLGlossary(url, callback, withMetaData) {
    if ($("#navbar a:contains('Login')").length > 0){
        return Promise.resolve(); // not logged in
    }
    withMetaData = withMetaData || false;
    if (!useLNMTLGlossary)
        callback = callback || function() { return Promise.resolve();};
    else
        callback = callback || applyGlossaryAsync;
    let novelData = restore("ndata" + url);
    let timeNow = new Date().getTime();
    if (novelData === null )
        novelData = { time: 0 , changes: 0 };

    let old_changes = novelData.changes;
    let old_retrans = novelData.retrans;
    let callGetLNMTLGLossary = function (novelData, old_changes, old_retrans, callback, withMetaData){
        return getLNMTLGlossary(novelData.novel_id, novelData.changes, ((novelData.changes - old_changes <= 0) && old_retrans === novelData.retrans), callback, withMetaData);
    };
    if (timeNow - novelData.time >= 3600000 * novelRefreshTime){
        console.log("fetch novel data: " + url);
        return new Promise( resolve => {
            fetchLNMTLNovelData(url, function (data) {
                let novelinfo = $('.panel.panel-default .progress-bar[role="progressbar"]', $(data));
                let glossary_changes = novelinfo.text().trim().split(' / ');
                let retranslation_time = new Date($(".panel.panel-default dt:contains('Latest retranslation at') ~ * span", $(data)).text()).getTime();
                let novel_id = $("a.btn.btn-primary[href^='https://lnmtl.com/termProposition?novel_id=']", $(data)).attr("href").split("https://lnmtl.com/termProposition?novel_id=")[1];
                novelData.time = timeNow;
                novelData.changes = glossary_changes[0];
                novelData.retrans = retranslation_time;
                novelData.novel_id = novel_id;
                store("ndata" + url, novelData);
                callGetLNMTLGLossary(novelData, old_changes, old_retrans, callback, withMetaData).then((value) => resolve(value));
            }).fail(e => resolve());
        });
    } else {
        return callGetLNMTLGLossary(novelData, old_changes, old_retrans, callback, withMetaData);
    }
}

function analyseGlossary(dict, metaDict, monitor){
    return new Promise(function(resolve, reject) {
        monitor = monitor || function() {};
        metaDict = metaDict || [];
        let reorder = {};
        let badorder = {};
        let goodorder = {};
        let duplicates = {};
        let whitespaces = [];
        let chunk = 100;
        let pcount = 0;
        let i = 0;

        function processTerms() {
            let cnt = chunk;
            for (;i < dict.length && ( cnt-- > 0); i++) {
                let term = dict[i];
                if (term.raw == "") //#skip empty raw
                    continue;
                let j = i + 1;
                let after_term_pos = i;
                while (j < dict.length){
                    if (i === j){
                        break;}
                    let bterm = dict[j];
                    //filter dublicate entries
                    if (term.raw == bterm.raw){
                        if (j > i)
                            duplicates[i] = [j];
                    }
                    else {
                        if (bterm.raw.includes(term.raw) && after_term_pos < j) {
                            if(j > i){
                                reorder[i] = j;
                                after_term_pos = j;
                                (badorder[i] || (badorder[i] = [])).push(j);
                            }
                            else{
                                (goodorder[i] || (goodorder[i] = [])).push(j);
                            }
                        }
                    }
                    if (++j == dict.length && after_term_pos != i){
                        after_term_pos = -1;
                        j = 0;
                    }

                }
                if (term.meaning.trim() != term.meaning)
                    whitespaces.push(term); // whitespaces removed during parsing!
            }
            if (pcount % 10 == 0)
                monitor(i, dict.length);
            if (i < dict.length) {
                pcount++;
                setTimeout(processTerms, 1);
            }
            else{
                monitor(100, 100); // progress complete
                let reverseDict = function(map, f) {
                    let id = function(x) {return x;};
                    return Object.keys(map).reduce(function(acc, k) {
                        acc[map[k]] = (acc[map[k]] || []).concat((f || id)(k));
                        return acc;
                    },{});
                };
                duplicates = reverseDict(duplicates);
                let conflicts = reverseDict(reorder);

                for(let key in conflicts){
                    if(!goodorder[key])
                        goodorder[key] = conflicts[key];
                }

                let metadata = {"terms": dict.length,"conflicts": Object.keys(reorder).length, "duplicates": Object.keys(duplicates).length, "whitespaces": whitespaces.length};
                let realPosition = (pos) => (metaDict[pos]||[parseInt(pos)+1])[0];
                let posWrapper = (term, pos) => { return {"raw":term.raw.replace(/\\\\/g,''), "meaning":term.meaning, "pos":realPosition(pos)}; };
                let infoWrapper = (term, pos) => { return {"raw":term.raw.replace(/\\\\/g,''), "meaning":term.meaning, "pos":realPosition(pos), "goodorder":termInfos(goodorder[pos], posWrapper), "badorder": termInfos(badorder[pos], posWrapper)}; };
                let termInfos = (terms, wrapper) => (terms||[]).reduce((p, i) => p.concat(wrapper(dict[i],i)), []);
                let conflictResult = Object.keys(conflicts).reduce((p, key) => {
                    return function(p, term, key, value) {
                        return p.concat([termInfos(value, infoWrapper).concat([infoWrapper(term, key)])]);
                    }(p, dict[key], key, conflicts[key]);
                }, []);

                let duplicatesResult = Object.keys(duplicates).reduce((p, key) => {
                    return function(p, term, key, value) {
                        return p.concat([termInfos(value, posWrapper).concat([posWrapper(term, key)])]);
                    }(p, dict[key], key, duplicates[key]);
                }, []);

                let result = {"metadata":metadata, "conflicts":conflictResult, "duplicates": duplicatesResult, "whitespaces":whitespaces};
                resolve(result);
            }
        }
        processTerms();

    });
}

function processArrayAsync(arr, fn, maxTimePerChunk, chunk, context) {
    context = context || window;
    maxTimePerChunk = maxTimePerChunk || 200;
    chunk = chunk || 50;
    let i = 0;

    function now() {
        return new Date().getTime();
    }

    function processChunk() {
        let startTime = now();
        let cnt = chunk;
        while (i < arr.length && ( cnt-- > 0 || (now() - startTime) <= maxTimePerChunk)) {
            if (cnt <= 0){
                cnt = chunk;
            }
            // callback called with (value, index, array)
            fn.call(context, arr[i], i++, arr);
        }
        if (i < arr.length) {
            setTimeout(processChunk, 1);
        }
    }
    processChunk();
}

function getLNMTLGlossary(novelid, changeCount, skipUpdate, callback, withMetaData){
    withMetaData = withMetaData || false;
    let glossaryData = (restore("gdata" + novelid));
    let glossaryMetaData;

    if(withMetaData && !restore("gvalid" + novelid, false)){
        removeSession("gmdata"+ novelid);
    }

    if (glossaryData === null || (withMetaData && ((glossaryMetaData = restoreSession("gmdata" + novelid)) === null))){
        glossaryData = [];
        glossaryMetaData = [];
        skipUpdate = false;
    }

    if (!skipUpdate){
        console.log("fetch novel glossary:" + novelid);
        return new Promise( resolve => {
            fetchLNMTLGlossary(novelid, function (data){
                let o = $('#after_term_id', $(data))[0].options;
                processArrayAsync(o, (v,i) => {
                    let g = (regexGlossaryTerms.exec(v.text) || []);
                    let dict = {};
                    dict.raw = (g[2]||"").trim().replace(/[!"#$%&'()*+,.\/:;<=>?@[\\\]^`{|}~]/g, "\\\\$&");
                    dict.meaning = (g[3]||"").trim();
                    //dict.id = v.value;
                    glossaryData[i] = dict;
                    if(withMetaData)
                        glossaryMetaData[i] = [parseInt(g[1]),parseInt(v.value)];
                });

                //glossaryData = $.map($('#after_term_id option', $(data)).get(), function( a ) { let dict = {}; return dict;});
                store("gdata" + novelid, glossaryData);

                if(withMetaData){
                    storeSession("gmdata" + novelid, glossaryMetaData);
                    store("gvalid" + novelid, true);   //indicate if sessionStorage and localStorage are consistent
                } else if (restore("gvalid" + novelid, false))
                    removeStorage("gvalid" + novelid); //invalidate sessionStorage on localStorage change

                //console.log("dict length ="+ glossaryData.length);
                callback(glossaryData, changeCount, glossaryMetaData, novelid).then((value) => resolve(value));
            }, resolve);
        });
    }
    else {
        //console.log("store dict length ="+ glossaryData.length);
        return callback(glossaryData, changeCount, glossaryMetaData, novelid);
    }
}


function fetchLNMTLGlossary(novelid, callback, resolve, term_id, trys) {
    if (!term_id)
        term_id = 1;
    if (!trys)
        trys = 10;
    $.get("https://lnmtl.com/termProposition/create?type=move&novel_id=" + novelid + "&term_id="+term_id, function(response) {
        callback(response);
    }).fail(function(e) { // workaround if id 1 is missing.
        if (e.status == 404) {
            if(trys > 0 || term_id < 4196)
                return fetchLNMTLGlossary(novelid, callback, resolve, term_id*2, trys-1);
            else
                resolve();
        } else {
            resolve();
        }
    });
}

String.prototype.escapeRegExp = function () {
    let regFilter = /[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g;
    return this.replace(regFilter, "\\$&");
};

function escapeRegExp(str) {
    let regFilter = /[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g;
    if (str.constructor === Array){
        let newStr = [];
        for(let i in str){
            newStr.push(str[i].replace(regFilter, "\\$&"));
        }
        return newStr;
    }
    return str.replace(regFilter, "\\$&");
}

function fetch_glossary() {
    Tabletop.init( { key: publicSpreadsheetUrl,
                    callback: useData,
                    simpleSheet: false } );
}

function updateReplace(replace){
    replaceInOriginal = replace;
    GM_SuperValue.set("repalceInOriginal", replaceInOriginal);
}

function updateDocsURL(url){
    if(!(url.startsWith("https://") || url.startsWith("http://") || url === ""))
        // no url no save
        return;
    publicSpreadsheetUrl = url;
    GM_SuperValue.set("publicSpreadsheetUrl", publicSpreadsheetUrl);
}

function updateUseLNMTLGlossary(_useLNMTLGlossary){
    useLNMTLGlossary = _useLNMTLGlossary;
    GM_SuperValue.set("useLNMTLGlossary", useLNMTLGlossary);

    if (window.location.pathname.startsWith("/chapter/") || window.location.pathname.startsWith("chapter/") && useLNMTLGlossary) {
        let novelname = window.location.pathname.split("/").pop();
        novelname = novelname.substr(0,novelname.lastIndexOf("-chapter-"));
        let book_index = novelname.lastIndexOf("-book-");
        if(book_index > novelname.length - 10 && book_index > 0){
            novelname = novelname.substr(0, book_index);
        }

        LNMTLGlossaryLock = lock(() => loadLNMTLGlossary("https://lnmtl.com/novel/" + novelname));
        fetch_glossary(); //google docs glossary
    }
}

function updateDisableGlossaryAnalyzer(_disableGlossaryAnalyzer){
    disableGlossaryAnalyzer = _disableGlossaryAnalyzer;
    GM_SuperValue.set("disableGlossaryAnalyzer", disableGlossaryAnalyzer);

    if (window.location.pathname.startsWith("/novel/") || window.location.pathname.startsWith("novel/") && !disableGlossaryAnalyzer) {
        if (!disableGlossaryAnalyzer)
            if ($('#novel-display-conflicts-button').length == 0){
                addGlossaryAnalyseModal();
                addAnalyseButton();
            }
            else
                $('#novel-display-conflicts-button').show();
        else $('#novel-display-conflicts-button').hide();
    }
}

function updateContrast(_reduceContrast){
    reduceContrast = _reduceContrast;
    GM_SuperValue.set("reduceContrast", reduceContrast);
    changeContrast(reduceContrast);
}

function updateWordWrap(_noWordWrap){
    noWordWrap = _noWordWrap;
    GM_SuperValue.set("noWordWrap", noWordWrap);
    changeWordWrap(noWordWrap);
}

function addCloseSettingsListener(){
    $("#chapter-display-options-modal div.modal-footer button").click(function(){$('#chapter-display-options-modal div.modal-body #gdocs-url-input').val(publicSpreadsheetUrl);});
}

function addUserGlossaryLink(){
    if(publicSpreadsheetUrl !== ""){
        // if the gdocs url links to the web version than move to edit mode
        let editurl = publicSpreadsheetUrl.replace("/pubhtml","/edit");
        let ex_icon = '<sup> <span class="glyphicon glyphicon-new-window"></span></sup>';
        let link = $('<li><a target="_blank" href="'+editurl+'">Google User Glossary'+ex_icon+'</a></li>');
        $('#navbar > ul.nav.navbar-nav.navbar-right > li.dropdown .dropdown-header:contains("Glossary") ~ .divider').before(link);
    }
}

function addGlossarySettings(){
    let title = $("<h3>Advanced Glossary</h3>");
    let row = $('<div class="row"></div>');
    let col = $('<div class="col-xs-6"></div>');
    let label = $('<label class="control-label">User Glossary Sheet Url:</label>');
    let textarea_row =$('<div class="col-xs-12"></div>');
    let textarea =$('<textarea class="form-control" id="gdocs-url-input" rows="2" input type="text" name="gdocs-url" wrap="soft">');
    textarea.val(publicSpreadsheetUrl);
    textarea.on("change keyup paste", function(){updateDocsURL(textarea.val());});
    textarea_row.append(textarea);
    col.append(label);
    let col2 = $('<div class="col-xs-6"></div>');
    let checked = "";
    if(replaceInOriginal)
        checked = "checked";
    let option2 = $('<sub><input id="replaceInOriginal" type="checkbox" '+checked+'></sub> <label for="replaceInOriginal">Replace all raws in source sentence</label>');
    option2.on("change", function(){updateReplace($("#replaceInOriginal")[0].checked);});
    col2.append(option2);
    row.append(col).append(col2).append(textarea_row);

    $("#chapter-display-options-modal .modal-body").append(title).append(row);

    if ($("#navbar a:contains('Login')").length > 0){
        // not logged in
        return;
    }

    let LNMTLGlossaryChecked = "";
    if (useLNMTLGlossary)
        LNMTLGlossaryChecked = "checked";

    let col3 = $('<div class="col-xs-12"></div>');
    let option3 = $('<sub><input id="useLNMTLGlossary" type="checkbox" '+LNMTLGlossaryChecked+'></sub> <label for="useLNMTLGlossary">Apply approved LNMTL Glossary entries to chapters. (cache refreshes every ' + novelRefreshTime+'h)</label>');
    option3.on("change", function(){updateUseLNMTLGlossary($("#useLNMTLGlossary")[0].checked);});
    let GlossaryAnalyzerChecked = "";
    if (disableGlossaryAnalyzer)
        GlossaryAnalyzerChecked = "checked";
    let col4 = $('<div class="col-xs-12"></div>');
    let option4 = $('<sub><input id="disableGlossaryAnalyzer" type="checkbox" '+GlossaryAnalyzerChecked+'></sub> <label for="disableGlossaryAnalyzer">Disable Glossary Analyser</label>');
    option4.on("change", function(){updateDisableGlossaryAnalyzer($("#disableGlossaryAnalyzer")[0].checked);});

    row.append(col3.append(option3)).append(col4.append(option4));
}

function createConflictReport(data, showPopover, caption){
    showPopover = showPopover || false;
    if (caption)
        caption = '<caption align="bottom">' + caption + '</caption>';
    else
        caption = "";
    let popdiv = $('<div id=popoverConflictTarget> </div>');
    let table = $('<table class="table table-sm table-condensed"> '+ caption + ' <thead> <tr> <th scope="col">raw</th> <th scope="col">meaning</th> <th scope="col">glossary position</th> </tr> </thead> </table>');
    let tdata = '';
    let delimiter = '<tr class="active"> <th scope="row" colspan="3"></th> </tr>';

    let addPopover = function(text, popover) {
        if(!showPopover)
            return text;
        return '<td tabindex="0" data-toggle="popover" data-trigger="focus" title="glossary order" data-delay="{ "show": 800, "hide": 400 }" data-placement="auto" data-content="'+popover+'" data-html="true" >' + text + '</td>';
    };

    let locationContext = function (term){
        let result = "<ul class='list-group'>";
        let good = term.goodorder;
        let bad = term.badorder;

        for(let key in good){
            result +="<li class='list-group-item'>"+ termPosition(good[key])+"</li>";
        }
        result +="<li class='list-group-item active'>"+ termPosition(term)+"</li>";
        for(let key in bad){
            result += "<li class='list-group-item'>"+ termPosition(bad[key])+"</li>";
        }
        return result + '</ul>';
    };

    let termPosition = function (term){
        return term.pos + " | " + term.raw;
    };

    for (i=0; i < data.length; i++ ) {
        for (j=0; j < data[i].length; j++ ) {
            let term = data[i][j];
            tdata += '<tr> <th scope="row">' + term.raw + '</th> ' + addPopover(term.meaning,locationContext(term)) + '<td>'+ term.pos + '</td> </tr>';
        }
        tdata += delimiter;
    }
    let tbody = $('<tbody>' + tdata + '</tbody>');
    table.append(tbody);
    return popdiv.append(table);
}

function createTermGroupReport(data, caption){
    if (caption)
        caption = '<caption align="bottom">' + caption + '</caption>';
    else
        caption = "";
    let table = $('<table class="table" table-sm table-condensed> ' + caption + ' <thead> <tr> <th scope="col">raw</th> <th scope="col">meaning</th> <th scope="col">position</th> </tr> </thead>');
    let tdata = '';
    let delimiter = '<tr class="active"> <th scope="row" colspan="3"></th> </tr>';

    for (i=0; i < data.length; i++ ) {
        for (j=0; j < data[i].length; j++ ) {
            let term = data[i][j];
            tdata += '<tr> <th scope="row">' + term.raw + '</th> <td>' + term.meaning + '</td> <td>'+ term.pos + '</td> </tr>';
        }
        tdata += delimiter;
    }

    let tbody = $('<tbody>' + tdata + '</tbody>');
    table.append(tbody);
    return table;
}

function createTermReport(data,caption){
    if (caption)
        caption = '<caption align="bottom">' + caption + '</caption>';
    else
        caption = "";
    let table = $('<table class="table table-sm table-condensed">' + caption + ' <thead> <tr> <th scope="col">raw</th> <th scope="col">meaning</th> </tr> </thead>');
    let tdata = '';

    for (i=0; i < data.length; i++ ) {
        let term = data[i];
        tdata += '<tr> <th scope="row">' + term.raw + '</th> <td>' + term.meaning + '</td> </tr>';
    }

    let tbody = $('<tbody>' + tdata + '</tbody>');
    table.append(tbody);
    return table;
}

function createAnalyzerReport(data){
    let intro = $('<p></p>');
    let metadata = data.metadata;
    let plural = 's';

    let info1 = '<h5><span class="label label-default">'+metadata.conflicts+'</span> of <span class="label label-default">'+metadata.terms+'</span> terms out of order</h5>' ;
    if (metadata.duplicates == 1)
        plural = '';
    let info2 = '<h5><span class="label label-default">'+metadata.duplicates+'</span> duplicate'+plural+' detected</h5>';
    plural = 's';
    if (metadata.whitespaces == 1)
        plural = '';
    let info3 = '<h5><span class="label label-default">'+ metadata.whitespaces+'</span> term'+plural+' with misplaced whitespaces</h5>';

    let head = intro.append($(info1));
    if(metadata.conflicts != 0){
        let conflictReport = createConflictReport(data.conflicts, true, "conflicting groups of terms");
        head.append(conflictReport);
    }
    if(metadata.duplicates != 0){
        if(metadata.conflicts != 0)
            head.append($('<hr>'));
        let duplicateReport = createTermGroupReport(data.duplicates, "duplicate terms");
        head.append(info2).append(duplicateReport);
    }
    if(metadata.whitespaces != 0){
        if(metadata.conflicts != 0 || metadata.duplicates != 0)
            head.append($('<hr>'));
        let whitespaceReport = createTermReport(data.whitespaces, "terms with leading or trailing whitespace");
        head.append(info3).append(whitespaceReport);
    }
    return head;
}

function addAnalyseButton(){
    let firstBtn = $("a.btn.btn-primary[href^='https://lnmtl.com/termProposition?novel_id=']:contains('Check New Term Propositions')" );
    firstBtn.after($('<button type="button" id="novel-display-conflicts-button" class="btn btn-warning" data-toggle="modal" data-target="#novel-display-conflicts-modal"> Analyse Glossary</button>'));
}

function addGlossaryAnalyseModal() {
    //anylyse glossary
    // conflics, duplicates, whitespace terms
    let novelname = $('.novel-name')[0].firstChild.textContent.trim();
    let module = $('<div class="my-module"> <div class="modal fade" id="novel-display-conflicts-modal"> <div class="modal-dialog"> <div class="modal-content"> </div> </div> </div> </div>');
    let modal_header = $('<div class="modal-header"> <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button> <h4 class="modal-title"> Analyse <em>' + novelname + '</em> Glossary</h4> </div>');
    let modal_body = $('<div class="modal-body"></div>');
    let progress_bar = $('<div class="collapse" id="collapseProgressbar"> <div class="progress"> <div class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" id="progress-bar-scan" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width: 0%; min-width: 3em;"> Scan Glossary </div> </div> </div>');
    let modal_footer = $('<div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>');
    modal_body.append(progress_bar);
    $('.modal-content', module).append(modal_header).append(modal_body).append(modal_footer);
    $( "#chapter-display-options-modal" ).after(module);

    let pMonitor = (value, maxvalue) => progressMonitor($('#progress-bar-scan', module), value, maxvalue);
    pMonitor(0,100);

    $('#novel-display-conflicts-modal').on('shown.bs.modal', scan = function(e) {
        $('#novel-display-conflicts-modal').off('shown.bs.modal', scan);
        loadLNMTLGlossary(window.location.toString(), function (dict, changes, metaDict) { return Promise.resolve(analyseGlossary(dict, metaDict, pMonitor));}, true)
            .then((results) => modal_body.append(createAnalyzerReport(results)))
            .then(() => setTimeout(() => $('#collapseProgressbar').collapse('hide'),1000))
            .then(() => $('#novel-display-conflicts-modal [data-toggle="popover"]').popover({container: "#popoverConflictTarget"}));
    });

    $('#novel-display-conflicts-modal').on('hidden.bs.modal', () => $('#novel-display-conflicts-button').focus(function() { this.blur();}));

    $('#chapter-display-conflicts-modal > .modal').modal("show");
    $('#collapseProgressbar').collapse();
    //addAnalyseBotton();
}

function progressMonitor(progressBar, value, maxvalue) {
    let pvalue = ~~(value / maxvalue * 100);
    progressBar.css('width', pvalue+'%').attr('aria-valuenow', pvalue);
}

function addStyleSettings() {
    let reduceContrastChecked = "";
    let noWordWrapChecked = "";
    if(reduceContrast)
        reduceContrastChecked = "checked";
    if(noWordWrap)
        noWordWrapChecked = "checked";

    let title = $("<h3>Styling</h3>");
    let row = $('<div class="row"></div>');
    let col = $('<div class="col-xs-6"></div>');
    let option = $('<sub><input id="reduceContrast" type="checkbox" '+reduceContrastChecked+'></sub> <label for="reduceContrast"> Reduce contrast of comment info box</label>');
    option.on("change", function(){updateContrast($("#reduceContrast")[0].checked);});
    let col2 = $('<div class="col-xs-6"></div>');
    let option2 = $('<sub><input id="noWordWrap" type="checkbox" '+noWordWrapChecked+'></sub> <label for="noWordWrap">Do not wrap words </label>');
    option2.on("change", function(){updateWordWrap($("#noWordWrap")[0].checked);});
    col.append(option);
    col2.append(option2);
    row.append(col2);
    if($('#chapter-display-options-modal div.modal-body select.form-control[data-type=backgroundColor]').val() === "dark")
        row.append(col);
    $("#chapter-display-options-modal .modal-body").append(title).append(row);
}

function addGlobalStyle(css) {
    let head, style;
    head = document.getElementsByTagName('head')[0];
    if (!head) { return; }
    style = document.createElement('style');
    style.type = 'text/css';
    style.innerHTML = css;
    head.appendChild(style);
}

function smartGlossary(dict){
    if(rawsReplaced)
        return;
    let exit = false;
    let ofRegExp = /的/g;
    let smartDict = [];
    for(let i in dict){
        let smartReplace = false;
        $('.original:contains("'+dict[i].raw+'")').each(function(){
            let dictRaw = dict[i].raw;
            let raw = this.innerText;
            let original = $(this);
            let translation = original.prev('.translated');
            let entry = new RegExp(dictRaw, 'g');
            let numOriginal = raw.match(entry).length;
            let traw = "";
            translation.find('w,t').each(function(){
                traw = traw + $(this).attr( 'data-title' );
            });
            let numTranslation = traw.count(entry);
            let trawClean = traw.replace(entry,"");
            if(numOriginal !== numTranslation){
                let anchorChars = "";
                let prependChars = "";
                let appendChars = "";
                //check if only "的" were omitted
                let testRaw = dictRaw.replace(ofRegExp,"");
                if(dictRaw.includes("的") && numOriginal === traw.count(testRaw) && raw.count(testRaw) === 0){
                    if(traw.count(ofRegExp) + numOriginal <= raw.count(ofRegExp))
                        anchorChars = testRaw;
                    dictRaw = testRaw;
                }
                else{
                    let xpendMissing = numOriginal - numTranslation;
                    for(let j = 0; j < dictRaw.length; j++)
                    {
                        let char = dictRaw.charAt(j);
                        let charReg = new RegExp(anchorChars + dictRaw.charAt(j), 'g');
                        let numOrigChar = raw.count(charReg);
                        //console.log( "#O " + numOriginal + " #OC " +numOrigChar);
                        if (numOriginal === numOrigChar){ //use as possible anchor
                            if (numOriginal === trawClean.count(charReg) + numTranslation){ //is likely safe anchor
                                anchorChars += char;
                                continue;
                            }
                        }

                        //check if xpend chars are missing or just relocated.
                        if(traw.count(char) + xpendMissing > raw.count(char)){
                            anchorChars=""; //remove anchor, unsafe xpend!
                            //console.log((traw.count(char) + xpendMissing) + ">" + (raw.count(char))+ " | " + xpendMissing + " ! " + dictRaw +" : "+char+ "\n" + raw);
                            break;
                        }

                        if(anchorChars.length === 0)
                            prependChars += char;
                        else{
                            //if not anchor char it has to be appended
                            appendChars += char;
                            //console.log("_"+anchorChars+"$ "+prependChars +" | "+ char);
                        }

                    }
                }
                let safetyCheck = new RegExp("([^" + escapeRegExp(prependChars) + "]|^)" + escapeRegExp(anchorChars) + "([^" + escapeRegExp(appendChars) + "]|$)", 'g');
                let safetySize = traw.count(safetyCheck);
                if((prependChars.length > 0 || appendChars.length > 0 || dictRaw !== dict[i].raw) && anchorChars.length > 0 && safetySize === (numOriginal - numTranslation)){
                    smartReplace = true;
                    let type = dict[i].overwrite === "TRUE" ? '':'w';
                    translation.find(type+'[data-title^='+anchorChars.charAt(0)+']').each(function(){
                        let node = $(this);
                        let rawattr = node.attr("data-title");
                        let toCleanup = [];
                        if(dict[i].overwrite === "TRUE"){
                            type = 'w, t';
                        }
                        while(rawattr.length < anchorChars.length && node.next(type).size() > 0){
                            let nextNode = node[0].nextSibling; //to check later for textnodes
                            node = node.next(type);
                            rawattr += node.attr("data-title");
                            // skip fill words from translation like "the", "can", whitespaces...
                            while (nextNode && nextNode.nodeType == 3) {
                                toCleanup.push(nextNode);
                                nextNode = node.nextSibling;
                            }
                            toCleanup.push(node);
                        }
                        if(rawattr == anchorChars){
                            let textPlaceholder = "";
                            for(let j in toCleanup){
                                textPlaceholder += $(toCleanup[j]).text();
                                toCleanup[j].remove();
                            }
                            if(textPlaceholder.length >0)
                                $(this).text($(this).text() + textPlaceholder);
                            $(this).attr('data-title', dict[i].raw);
                        }
                    });
                }
                //console.log(numOriginal + " <> " + numTranslation + ": " + dictRaw + " || " + anchorChars + " + " + appendChars);
            }
        });

        for(let j in smartDict)
        {
            if(smartReplace === true)
                break;
            if (dict[i].raw.includes(smartDict[j].raw))
                smartReplace = true;
        }
        if(smartReplace === true)
            smartDict.push(dict[i]);

    }

    //replace terms for "fixed" Raws
    replaceTerms(smartDict);
}

function rawsReplace(){
    if(replaceInOriginal) {
        $('.original t').each(function(){let mytext = $(this).text(); $(this).text($(this).attr('data-title').replace(/\(\w+\)/g,'')); $(this).attr('data-title', mytext);});
        // add whitespace between two english words
        $('.original').find('t').filter( function(index){var prev = $(this).get(0).previousSibling; return prev ? $(this).get(0).previousSibling.nodeName == 'T' : false;}).each(function(){$(this).text(' ' + $(this).text());});
        rawsReplaced = true;
    }
}

const OVERWRITE = {overwrite:"TRUE"};
function applyGlossaryAsync(dict, changeCount){
    if (autoSolveConflicts)
        dict = dict.map( item => { return {...item, ...OVERWRITE}; });
    if (changeCount == 0)
        dict = [];
    dict = dict.slice(-changeCount);
    return new Promise( resolve => {
        setTimeout(async function(){
            for(i = 0; i*glossaryChunkSize < dict.length; i++){
                applyGlossary(dict.slice(i*glossaryChunkSize, (i+1)*glossaryChunkSize), true);
                if ((i+1)*glossaryChunkSize < dict.length)
                    await sleep(300);
            }
            resolve();
        }, 1);
    }).then(() => console.log("Latest LNMTL glossary entries applied (" + dict.length + " terms)"));
}

function waitApplyGlossary(dict, skipReplace){
    return new Promise( resolve => {
        LNMTLGlossaryLock.then(() => {
            applyGlossary(dict, skipReplace);
            resolve();
        });
    });
}

function applyGlossary(dict, skipReplace){
    replaceTerms(dict);
    smartGlossary(dict);
    // original replace
    if(replaceInOriginal && !skipReplace && !rawsReplaced) {
        for(let i in dict){
            $('.original:contains("'+dict[i].raw+'")').find(':contains("'+dict[i].raw+'")').andSelf().contents().each(function() {
                let text = $(this).text();
                if(this.nodeType == 3){
                    $(this).replaceWith(text.replace(new RegExp(dict[i].raw, "g"), ' '+dict[i].meaning+' '));
                }
                else if(dict[i].overwrite === "TRUE" && $(this).is('t')){
                    if($(this).text() == dict[i].raw)
                        $(this).attr('data-title', dict[i].meaning);
                }
            });
        }
    }
}

function replaceTerms(dict){
    let simplereplace = false;
    for(let i in dict){
        //$('.translated').find('w[data-title='+dict[i].raw+']').text(dict[i].meaning);
        dict[i].raw = dict[i].raw.trim();
        if(dict[i].raw.length === 0){
            continue;
        }
        let type = 'w';
        if(dict[i].overwrite === "TRUE"){
            type = '';
        }
        $('.translated '+type+'[data-title^='+dict[i].raw.charAt(0)+']').each(function(){
            let node = $(this);
            let raw = node.attr("data-title");
            if(simplereplace && raw == dict[i].raw){
                node.text(dict[i].meaning);
                return;
            }
            let toCleanup = [];
            if(dict[i].overwrite === "TRUE"){
                type = 'w, t';
            }
            while(raw.length < dict[i].raw.length && node.next(type).size() > 0){
                let nextNode = node[0].nextSibling; //to check later for textnodes
                node = node.next(type);
                // console.log(node.text() + node.attr("data-title"));
                raw += node.attr("data-title");
                // skip fill words from translation like "the", "can", whitespaces...
                while (nextNode && nextNode.nodeType == 3) {
                    // console.log("textnode: " + nextNode.data);
                    toCleanup.push(nextNode);
                    nextNode = node.nextSibling;
                }

                toCleanup.push(node);
            }
            if(raw == dict[i].raw){
                $(this).text(dict[i].meaning);
                for(let j in toCleanup){
                    toCleanup[j].remove();
                }
                let text = dict[i].meaning;
                if(this.previousSibling.nodeValue === ". " || $(this).prev().size() === 0 && ($(this).parent('sentence').size() > 0 || ($(this).parent('dq').size() > 0 && $(this).parent('dq').prev().size() === 0 ))){
                    text = capitalize(text);
                }
                let translate = $("<t></t>").text(text);
                translate.attr("data-title", dict[i].raw);
                $(this).after(translate);
                $(this).remove();
                //fix click
                translate.click(function(){ $('t').popover({trigger: 'click', placement: 'top', html : true });});
            }
        });
    }
}

function novelSheet(tabletop, novelname){
    let sheetnames = tabletop.modelNames.join('|').toLowerCase().replace(/ /g,"-").split('|');
    let sheetindex = sheetnames.indexOf(novelname);
    if (sheetindex !== -1) {
        let sheetname = tabletop.modelNames[sheetindex];
        if(tabletop.sheets(sheetname) !== undefined){
            return waitApplyGlossary(tabletop.sheets(sheetname).elements).then(() => console.log("Applied glossary: " + sheetname));
        }
    }
    return Promise.resolve();
}

function useData(data, tabletop) {
    let sheets = Object.keys(data);
    if(sheets.length === 1){
        applyGlossary(tabletop.sheets(sheets[0]).elements);
        return;
    }

    let novelname = window.location.pathname.split("/").pop();
    novelname = novelname.substr(0,novelname.lastIndexOf("-chapter-"));
    let book_index = novelname.lastIndexOf("-book-");
    if(book_index > novelname.length - 10 && book_index > 0){
        novelname = novelname.substr(0, book_index);
    }

    novelSheet(tabletop, novelname).then(() => novelSheet(tabletop, "global")).then(rawsReplace());
}

function applySetting(){
    changeWordWrap(noWordWrap);
    changeContrast(reduceContrast);
}

function addCustomStyles(){
    // no word wrapping
    addGlobalStyle('body .chapter-body.noWordWrapping { hyphens: none;}');
    // reduced contrast (of comment box)
    addGlobalStyle('.reducedContrast.backgroundColorDark .alert-info , .reducedContrast.backgroundColorBlack .alert-info { background-color: #2d323f; border-color: #145461}');
    // bugfixes
    addGlobalStyle('.backgroundColorBright .tt-input  { color: #464444 !important;}');
    addGlobalStyle('.fontColorGray .chapter-body w.postfixed { color: #95a597 !important; }');
    addGlobalStyle('.fontColorGray .chapter-title {color: #9b9b9b;}');
    // glossary analyzer
    addGlobalStyle('.progress-bar-animated {animation: progress-bar-stripes 1s linear infinite;}');
    addGlobalStyle('.backgroundColorBlack #novel-display-conflicts-modal .table>tbody>tr.active>td, .backgroundColorBlack #novel-display-conflicts-modal .table>tbody>tr.active>th { background-color: #DDD !important; padding: 1px !important;}');
    addGlobalStyle('.backgroundColorDark #novel-display-conflicts-modal .table>tbody>tr.active>td, .backgroundColorDark #novel-display-conflicts-modal .table>tbody>tr.active>th { background-color: #DDD !important; padding: 1px !important;}');

}

function changeWordWrap(on){
    if(on === true)
        $('body .chapter-body').addClass("noWordWrapping");
    if(on === false)
        $('body .chapter-body').removeClass("noWordWrapping");
}

function changeContrast(on){
    if(on === true)
        $('body').addClass("reducedContrast");
    if(on === false)
        $('body').removeClass("reducedContrast");
}

(function() {
    'use strict';
    addCustomStyles();
    if (!disableGlossaryAnalyzer && $("#navbar a:contains('Login')").length == 0)
        addAnalyseButton();
    applySetting();
    addCloseSettingsListener();
    addUserGlossaryLink();
    addGlossarySettings();
    addStyleSettings();


    if ((window.location.pathname.startsWith("/novel/") || window.location.pathname.startsWith("novel/"))
        && ($("#navbar a:contains('Login')").length == 0 && (useLNMTLGlossary || !disableGlossaryAnalyzer))) {
        let data = ":root";
        let url = window.location.toString();
        let novelData = restore("ndata" + url);
        let timeNow = new Date().getTime();
        if (novelData === null )
            novelData = { changes: 0 };
        console.log("update novel metadata: " + url);
        let old_changes = novelData.changes;
        let old_retrans = novelData.retrans;
        let novelinfo = $('.panel.panel-default .progress-bar[role="progressbar"]', $(data));
        let glossary_changes = novelinfo.text().trim().split(' / ');
        let retranslation_time = new Date($(".panel.panel-default dt:contains('Latest retranslation at') ~ * span", $(data)).text()).getTime();
        let novel_id = $("a.btn.btn-primary[href^='https://lnmtl.com/termProposition?novel_id=']", $(data)).attr("href").split("https://lnmtl.com/termProposition?novel_id=")[1];
        novelData.time = timeNow;
        novelData.changes = glossary_changes[0];
        novelData.retrans = retranslation_time;
        novelData.novel_id = novel_id;
        store("ndata" + url, novelData);
        if (novelData.changes - old_changes != 0 || old_retrans !== novelData.retrans)
            localStorage.removeItem("gdata" + novelData.novel_id);
        if (!disableGlossaryAnalyzer)
            addGlossaryAnalyseModal();
    }


    if (window.location.pathname.startsWith("/chapter/") || window.location.pathname.startsWith("chapter/")){
        let novelname = window.location.pathname.split("/").pop();
        novelname = novelname.substr(0,novelname.lastIndexOf("-chapter-"));
        let book_index = novelname.lastIndexOf("-book-");
        if(book_index > novelname.length - 10 && book_index > 0){
            novelname = novelname.substr(0, book_index);
        }

        if (useLNMTLGlossary)
            LNMTLGlossaryLock = lock(() => loadLNMTLGlossary("https://lnmtl.com/novel/" + novelname));

        fetch_glossary(); //google docs glossary
    }
})();