/*
 * This file is part of the Amalthea library.
 * Copyright (C) 2018-2021, 2024-2026 Eugene 'Vindex' Stulin
 * Distributed under the BSL 1.0 or the GNU LGPL 3.0 or later.
 */

/**
 * Some useful functions based on std.net.curl.
 */
module amalthea.net;

static import core.time;
import core.time : msecs;

import std.algorithm : canFind, count, startsWith;
import std.array : appender;
import std.conv : to, parse;
import std.format : format;
static import std.net.curl;
import std.net.curl : HTTP, HTTPStatusException;
import std.range : empty, replace, split;
static import std.regex;
import std.regex : Captures;
import std.string : strip, toLower, toUpper;
static import std.utf;
import std.typecons : Tuple;

public import amalthea.libcore;
static import amalthea.dataprocessing;
import amalthea.dataprocessing :
    removeDuplicateConsecutiveSubstring, urlEncode;
import amalthea.encoding : UniString;


/// Default user agent for network requests.
enum defaultUserAgent =
    "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";


/*******************************************************************************
 * Exception for common errors in this module.
 */
class AmaltheaNetException : Exception {
    this(string msg, string file = __FILE__, size_t line = __LINE__) {
        super(msg, file, line);
    }
}


/*******************************************************************************
 * Simple GET request to raw content.
 * Returns: Byte array.
 */
deprecated ubyte[] getRaw(string url) {
    return get(url);
}


/*******************************************************************************
 * GET request to raw content.
 * Params:
 *     url                 = Resource address.
 *     timeoutMilliseconds = Request execution timeout in milliseconds.
 *                           If -1, timeout is not used.
 *     userAgent           = User agent.
 * Returns: Byte array.
 */
ubyte[] get(
    string url,
    long timeoutMilliseconds = -1,
    string userAgent = defaultUserAgent
) {
    auto client = HTTP();
    client.verifyPeer = false;
    client.setUserAgent(userAgent);
    if (timeoutMilliseconds != -1) {
        client.operationTimeout(timeoutMilliseconds.msecs);
    }
    return get(url, client);
}


/*******************************************************************************
 * GET request to raw content.
 * Params:
 *     url    = Resource address.
 *     client = Prepared std.net.curl.HTTP structure.
 * Returns: Byte array.
 */
ubyte[] get(string url, HTTP client) {
    client.method = HTTP.Method.get;
    client.url = url;
    HTTP.StatusLine statusLine;
    auto content = appender!(ubyte[])();
    client.onReceive = (ubyte[] data) {
        content ~= data;
        return data.length;
    };
    client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
    client.perform();
    if (statusLine.code / 100 != 2) {
        auto exceptionMsg = format!"HTTP request returned status code %d (%s)"(
            statusLine.code, statusLine.reason
        );
        throw new HTTPStatusException(statusLine.code, exceptionMsg);
    }
    return content.data;
}


/*******************************************************************************
 * POST request.
 */
T[] post(T = char)(
    const(char)[] url, string[string] postDict, HTTP conn = HTTP()
) if (is(T == char) || is(T == ubyte)) {
    return std.net.curl.post!T(url, urlEncode(postDict), conn);
}


/*******************************************************************************
 * POST request to get an answer with raw content.
 * Returns: Byte array.
 */
ubyte[] postRaw(const(char)[] url, string[string] postDict, HTTP c = HTTP()) {
    return amalthea.net.post!ubyte(url, postDict, c);
}


/*******************************************************************************
 * Gets HTTP response headers by URL.
 */
string[string] getHeaders(in char[] url) {
    auto http = HTTP(url);
    http.method = HTTP.Method.head;
    http.onReceive = (ubyte[] data) {
        return data.length;
    };
    http.perform();
    return http.responseHeaders;
}


/*******************************************************************************
 * Gets HTTP content type by URL.
 */
string getContentType(in char[] url) {
    auto header = getHeaders(url);
    string contentType;
    foreach(key, value; header) {
        if (key.toLower == "content-type") {
            contentType = value.strip.idup;
            break;
        }
    }
    return contentType;
}


/*******************************************************************************
 * Gets content charset (possibly empty) by URL.
 */
string getCharset(const(char)[] url) {
    string contentType = getContentType(url);
    string charsetName = extractCharsetFromContentType(contentType);
    return charsetName.empty ? "utf-8" : charsetName;
}


/*******************************************************************************
 * Checks if the URL is a link to HTML page.
 */
bool isLinkToHTML(string url) {
    try {
        string contentType = getContentType(url);
        if (contentType.canFind("text/html")) {
            return true;
        }
    } catch(Exception e) {
        return false;
    }
    return false;
}


/*******************************************************************************
 * Get text content as amalthea.encoding.UniString by URL.
 */
deprecated UniString getPage(string url) {
    return getText(url);
}


/*******************************************************************************
 * Get text content as amalthea.encoding.UniString by URL.
 * Params:
 *     url                 = Resource address.
 *     timeoutMilliseconds = Request execution timeout in milliseconds.
 *                           If -1, timeout is not used.
 *     userAgent           = User agent.
 * Returns:
 *     text response.
 */
UniString getText(
    string url,
    long timeoutMilliseconds = -1,
    string userAgent = defaultUserAgent
) {
    auto client = HTTP();
    client.verifyPeer = false;
    client.method = HTTP.Method.get;
    client.url = url;
    client.setUserAgent(userAgent);
    if (timeoutMilliseconds != -1) {
        client.operationTimeout(timeoutMilliseconds.msecs);
    }
    HTTP.StatusLine statusLine;
    auto content = appender!(ubyte[])();
    client.onReceive = (ubyte[] data) {
        content ~= data;
        return data.length;
    };
    string contentType;
    string charset;
    client.onReceiveHeader = (in char[] key, in char[] value) {
        auto lowerKey = key.idup.toLower;
        if (lowerKey == "content-length") {
            content.reserve(value.to!size_t);
        } else if (lowerKey == "content-type") {
            contentType = value.idup;
            charset = extractCharsetFromContentType(contentType);
        }
    };
    client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
    client.perform();

    if (statusLine.code / 100 != 2) {
        string exceptionMsg = format(
            "HTTP request returned status code %d (%s)",
            statusLine.code, statusLine.reason
        );
        throw new HTTPStatusException(statusLine.code, exceptionMsg);
    }
    if (charset.empty) {
        charset = "UTF-8";
    }
    auto page = UniString(charset, content.data);
    return page;
}



/*******************************************************************************
 * This function searches all elements from HTML page with a specific tag.
 */
auto getElementsByTag(string html, string tag) {
    return getElementsByTagAndAttribute(html, tag);
}


/*******************************************************************************
 * This function searches all elements by a specific tag and an attribute.
 * Returns: Page element info (tag name and possible attributes)
 *          and content of the element as tuple with two elements.
 */
auto getElementsByTagAndAttribute(
    string html,
    string tag,
    string attrName = "",
    string attrValue = ""
) {
    html = html.replace("\n", " ");
    auto lowerTag = tag.toLower;
    auto upperTag = tag.toUpper;
    html = html.replace("<"~upperTag~" ", "\n<"~lowerTag~" ")
               .replace("<"~upperTag~">", "\n<"~lowerTag~">")
               .replace("</"~upperTag~" ", "</"~lowerTag~" \n")
               .replace("</"~upperTag~">", "</"~lowerTag~">\n");
    html = html.replace("<"~lowerTag~" ", "\n<"~lowerTag~" ")
               .replace("<"~lowerTag~">", "\n<"~lowerTag~">")
               .replace("</"~lowerTag~" ", "</"~lowerTag~" \n")
               .replace("</"~lowerTag~">", "</"~lowerTag~">\n");
    if (!attrValue.empty) attrValue = ` *= *"?` ~ attrValue ~ `"?`;
    string openingTag, closingTag;
    openingTag = attrName.empty ? tag : tag~" ";
    closingTag = tag;
    string e;
    e = format!`^<(?P<declaration>%s[^>]*%s%s *[^>]*)>(?P<content>.*)</%s *>$`
        (openingTag, attrName, attrValue, closingTag);
    bool pairedTag = true;
    auto r = std.regex.regex(e, "im");
    if (count(std.regex.matchAll(html, r)) == 0) {
        e = format!`^<(?P<declaration>%s [^>]*%s%s *[^>]*)>`
                   (tag, attrName, attrValue);
        pairedTag = false;
        r = std.regex.regex(e, "im");
    }
    Tuple!(string, "declaration", string, "content")[] elements;
    foreach(c; std.regex.matchAll(html, r)) {
        auto decl = c["declaration"].replace("\t", " ");
        decl = decl.removeDuplicateConsecutiveSubstring(" ");
        elements ~= Tuple!(string, "declaration", string, "content")
                          (decl, pairedTag ? c["content"] : "");
    }
    return elements;
}


/*******************************************************************************
 * This function returns title of Internet-page.
 */
string getHTMLPageTitle(string url) {
    string html = getText(url, 10_000, defaultUserAgent).toString;
    html.replaceSpecialMnemonics;
    auto res = getElementsByTag(html, "title");
    if (res.empty) return "";
    return res[0].content;
}


/*******************************************************************************
 * Search and replace special characters in HTML for normal view.
 */
ref string replaceSpecialMnemonics(return ref string htmlText) {
    string[string] specialHTMLSymbols = [
        "&iexcl;"    : "¡",
        "&cent;"     : "¢",
        "&pound;"    : "£",
        "&curren;"   : "¤",
        "&yen;"      : "¥",
        "&brvbar;"   : "¦",
        "&sect;"     : "§",
        "&uml;"      : "¨",
        "&copy;"     : "©",
        "&ordf;"     : "ª",
        "&laquo;"    : "«",
        "&raquo;"    : "»",
        "&not;"      : "¬",
        "&shy;"      : "­",
        "&reg;"      : "®",
        "&macr;"     : "¯",
        "&deg;"      : "°",
        "&plusmn;"   : "±",
        "&sup2;"     : "²",
        "&sup3;"     : "³",
        "&acute;"    : "´",
        "&micro;"    : "µ",
        "&para;"     : "¶",
        "&middot;"   : "·",
        "&cedil;"    : "¸",
        "&sup1;"     : "¹",
        "&ordm;"     : "º",
        "&frac14;"   : "¼",
        "&frac12;"   : "½",
        "&frac34;"   : "¾",
        "&iquest;"   : "¿",
        "&Agrave;"   : "À",
        "&Aacute;"   : "Á",
        "&Acirc;"    : "Â",
        "&Atilde;"   : "Ã",
        "&Auml;"     : "Ä",
        "&Aring;"    : "Å",
        "&AElig;"    : "Æ",
        "&Ccedil;"   : "Ç",
        "&Egrave;"   : "È",
        "&Eacute;"   : "É",
        "&Ecirc;"    : "Ê",
        "&Euml;"     : "Ë",
        "&Igrave;"   : "Ì",
        "&Iacute;"   : "Í",
        "&Icirc;"    : "Î",
        "&Iuml;"     : "Ï",
        "&ETH;"      : "Ð",
        "&Ntilde;"   : "Ñ",
        "&Ograve;"   : "Ò",
        "&Oacute;"   : "Ó",
        "&Ocirc;"    : "Ô",
        "&Otilde;"   : "Õ",
        "&Ouml;"     : "Ö",
        "&times;"    : "×",
        "&Oslash;"   : "Ø",
        "&Ugrave;"   : "Ù",
        "&Uacute;"   : "Ú",
        "&Ucirc;"    : "Û",
        "&Uuml;"     : "Ü",
        "&Yacute;"   : "Ý",
        "&THORN;"    : "Þ",
        "&szlig;"    : "ß",
        "&agrave;"   : "à",
        "&aacute;"   : "á",
        "&acirc;"    : "â",
        "&atilde;"   : "ã",
        "&auml;"     : "ä",
        "&aring;"    : "å",
        "&aelig;"    : "æ",
        "&ccedil;"   : "ç",
        "&egrave;"   : "è",
        "&eacute;"   : "é",
        "&ecirc;"    : "ê",
        "&euml;"     : "ë",
        "&igrave;"   : "ì",
        "&iacute;"   : "í",
        "&icirc;"    : "î",
        "&iuml;"     : "ï",
        "&eth;"      : "ð",
        "&ntilde;"   : "ñ",
        "&ograve;"   : "ò",
        "&oacute;"   : "ó",
        "&ocirc;"    : "ô",
        "&otilde;"   : "õ",
        "&ouml;"     : "ö",
        "&divide;"   : "÷",
        "&oslash;"   : "ø",
        "&ugrave;"   : "ù",
        "&uacute;"   : "ú",
        "&ucirc;"    : "û",
        "&uuml;"     : "ü",
        "&yacute;"   : "ý",
        "&thorn;"    : "þ",
        "&yuml;"     : "ÿ",
        "&fnof;"     : "ƒ",
        "&Alpha;"    : "Α",
        "&Beta;"     : "Β",
        "&Gamma;"    : "Γ",
        "&Delta;"    : "Δ",
        "&Epsilon;"  : "Ε",
        "&Zeta;"     : "Ζ",
        "&Eta;"      : "Η",
        "&Theta;"    : "Θ",
        "&Iota;"     : "Ι",
        "&Kappa;"    : "Κ",
        "&Lambda;"   : "Λ",
        "&Mu;"       : "Μ",
        "&Nu;"       : "Ν",
        "&Xi;"       : "Ξ",
        "&Omicron;"  : "Ο",
        "&Pi;"       : "Π",
        "&Rho;"      : "Ρ",
        "&Sigma;"    : "Σ",
        "&Tau;"      : "Τ",
        "&Upsilon;"  : "Υ",
        "&Phi;"      : "Φ",
        "&Chi;"      : "Χ",
        "&Psi;"      : "Ψ",
        "&Omega;"    : "Ω",
        "&alpha;"    : "α",
        "&beta;"     : "β",
        "&gamma;"    : "γ",
        "&delta;"    : "δ",
        "&epsilon;"  : "ε",
        "&zeta;"     : "ζ",
        "&eta;"      : "η",
        "&theta;"    : "θ",
        "&iota;"     : "ι",
        "&kappa;"    : "κ",
        "&lambda;"   : "λ",
        "&mu;"       : "μ",
        "&nu;"       : "ν",
        "&xi;"       : "ξ",
        "&omicron;"  : "ο",
        "&pi;"       : "π",
        "&rho;"      : "ρ",
        "&sigmaf;"   : "ς",
        "&sigma;"    : "σ",
        "&tau;"      : "τ",
        "&upsilon;"  : "υ",
        "&phi;"      : "φ",
        "&chi;"      : "χ",
        "&psi;"      : "ψ",
        "&omega;"    : "ω",
        "&thetasym;" : "ϑ",
        "&upsih;"    : "ϒ",
        "&piv;"      : "ϖ",
        "&bull;"     : "•",
        "&hellip;"   : "…",
        "&prime;"    : "′",
        "&Prime;"    : "″",
        "&oline;"    : "‾",
        "&frasl;"    : "⁄",
        "&weierp;"   : "℘",
        "&image;"    : "ℑ",
        "&real;"     : "ℜ",
        "&trade;"    : "™",
        "&alefsym;"  : "ℵ",
        "&larr;"     : "←",
        "&uarr;"     : "↑",
        "&rarr;"     : "→",
        "&darr;"     : "↓",
        "&harr;"     : "↔",
        "&crarr;"    : "↵",
        "&lArr;"     : "⇐",
        "&uArr;"     : "⇑",
        "&rArr;"     : "⇒",
        "&dArr;"     : "⇓",
        "&hArr;"     : "⇔",
        "&forall;"   : "∀",
        "&part;"     : "∂",
        "&exist;"    : "∃",
        "&empty;"    : "∅",
        "&nabla;"    : "∇",
        "&isin;"     : "∈",
        "&notin;"    : "∉",
        "&ni;"       : "∋",
        "&prod;"     : "∏",
        "&sum;"      : "∑",
        "&minus;"    : "−",
        "&lowast;"   : "∗",
        "&radic;"    : "√",
        "&prop;"     : "∝",
        "&infin;"    : "∞",
        "&ang;"      : "∠",
        "&and;"      : "∧",
        "&or;"       : "∨",
        "&cap;"      : "∩",
        "&cup;"      : "∪",
        "&int;"      : "∫",
        "&there4;"   : "∴",
        "&sim;"      : "∼",
        "&cong;"     : "≅",
        "&asymp;"    : "≈",
        "&ne;"       : "≠",
        "&equiv;"    : "≡",
        "&le;"       : "≤",
        "&ge;"       : "≥",
        "&sub;"      : "⊂",
        "&sup;"      : "⊃",
        "&nsub;"     : "⊄",
        "&sube;"     : "⊆",
        "&supe;"     : "⊇",
        "&oplus;"    : "⊕",
        "&otimes;"   : "⊗",
        "&perp;"     : "⊥",
        "&sdot;"     : "⋅",
        "&lceil;"    : "⌈",
        "&rceil;"    : "⌉",
        "&lfloor;"   : "⌊",
        "&rfloor;"   : "⌋",
        "&lang;"     : "〈",
        "&rang;"     : "〉",
        "&loz;"      : "◊",
        "&spades;"   : "♠",
        "&clubs;"    : "♣",
        "&hearts;"   : "♥",
        "&diams;"    : "♦",

        "&apos;"     : "'",
        "&quot;"     : "\"",
        "&amp;"      : "&",
        "&lt;"       : "<",
        "&gt;"       : ">",
        "&OElig;"    : "Œ",
        "&oelig;"    : "œ",
        "&Scaron;"   : "Š",
        "&scaron;"   : "š",
        "&Yuml;"     : "Ÿ",
        "&circ;"     : "ˆ",
        "&tilde;"    : "˜",
        "&ndash;"    : "–",
        "&mdash;"    : "—",
        "&lsquo;"    : "‘",
        "&rsquo;"    : "’",
        "&sbquo;"    : "‚",
        "&ldquo;"    : "“",
        "&rdquo;"    : "”",
        "&bdquo;"    : "„",
        "&dagger;"   : "†",
        "&Dagger;"   : "‡",
        "&permil;"   : "‰",
        "&lsaquo;"   : "‹",
        "&rsaquo;"   : "›",
        "&euro;"     : "€",
    ];
    foreach(k, v; specialHTMLSymbols) {
        htmlText = htmlText.replace(k, v);
    }

    string decode(Captures!string cs) {
        string s = cs[1];  // without "&#" and ";"
        try {
            uint codePoint;
            if (s[0] == 'x' && s.length > 1) {
                s = s[1 .. $];
                codePoint = parse!uint(s, 16);
            } else {
                codePoint = to!uint(s);
            }
            return std.utf.toUTF8([cast(dchar) codePoint]);
        } catch (Exception e) {
            return s;
        }
    }

    auto numRefRegex = std.regex.regex(`&#(x[0-9a-fA-F]+|\d+);`);
    htmlText = std.regex.replaceAll!decode(htmlText, numRefRegex);

    return htmlText;
}
unittest {
    auto text = "x &le; 3 &#x2F; x &ge; &minus;2";
    assert(replaceSpecialMnemonics(text) == "x ≤ 3 / x ≥ −2");
}


private string extractCharsetFromContentType(string contentType) {
    string charset;
    if (!contentType.empty) {
        auto fields = contentType.split(';');
        foreach(field; fields) {
            field = field.strip;
            if (field.startsWith("charset=")) {
                charset = field["charset=".length .. $].idup;
                break;
            }
        }
    }
    return charset;
}
