-- TOG Tools — Item Database builder (engine) [developer tool]
--
-- Walks the entire item-ID space and records every item the client knows
-- into a saved name→link database, bucketed by WoW's item class (classID).
-- This is the data source for an offline "type a name, get the link" search
-- and — longer term — a shared item DB for the TOG suite.
--
-- Why a runtime walk: WoW ships no searchable item table and no name→item
-- API. The only way to build one offline is to ask the server about every
-- item ID once. `GetItemInfoInstant` makes that cheap and safe — it's a LOCAL,
-- synchronous call that returns the itemID (and classID / subClassID /
-- equipLoc) for REAL items and nil for non-existent IDs, so we only fire a
-- server request (`GetItemInfo`, for name/quality) for IDs that are real but
-- uncached, and pick up the answers via GET_ITEM_INFO_RECEIVED. Same idea the
-- "Get Link" / Ludwig addons use. (We deliberately do NOT use
-- C_Item.DoesItemExistByID — on Classic Era it returns true for ~every ID in
-- range, which would request and "pend" the entire ID space.)
--
-- Categorisation comes from that same GetItemInfoInstant call, so every item
-- is filed under its class bucket the moment we see it — name and quality
-- (which DO need the async fetch) are filled in alongside.
--
-- Storage (db.global.itemDB):
--   classes = { [classID] = { [idStr] = "name\31quality\31subClassID\31equipLoc" } }
--   count, cursor, topID, locale, build, complete, builtAt
--
-- This is a dev tool: it's flagged `devOnly` so the tab only appears when
-- Settings > General > Developer Tools is enabled.

local _, addon = ...

local ItemDB = {}
addon.modules["itemdb"] = ItemDB

-- ---------------------------------------------------------------------------
-- API shims — TBC+ namespaced these under C_Item; Classic Era keeps the bare
-- globals. Feature-detect so the same file runs on every flavour.
--
-- Existence test: we use GetItemInfoInstant, NOT DoesItemExistByID. On Classic
-- Era DoesItemExistByID returns true for essentially every ID in range, which
-- flooded `pending` with ~216k non-existent IDs and fired a request for each.
-- GetItemInfoInstant reads the client's static item data and returns the
-- itemID (and classID/subClassID/equipLoc) for REAL items only — nil for
-- non-existent IDs — so it's both the reliable existence gate and the source
-- of the category fields, in one synchronous call.
-- ---------------------------------------------------------------------------
local _GetItemInfoInstant  = (C_Item and C_Item.GetItemInfoInstant)  or GetItemInfoInstant
local _GetItemInfo         = (C_Item and C_Item.GetItemInfo)         or GetItemInfo
local _GetItemClassInfo    = (C_Item and C_Item.GetItemClassInfo)    or _G.GetItemClassInfo
local _GetItemSubClassInfo = (C_Item and C_Item.GetItemSubClassInfo) or _G.GetItemSubClassInfo

-- ---------------------------------------------------------------------------
-- Tuning
-- ---------------------------------------------------------------------------
-- WALK_PER_TICK: IDs advanced per tick. The per-ID existence/category check
--   (GetItemInfoInstant) is a cheap local call, so this can be high — it only
--   costs CPU, not server traffic.
-- REQ_PER_TICK: NEW server requests (real, uncached items) issued per tick.
--   This is the real throttle — capping it bounds the request burst that can
--   disconnect the client. ~REQ_PER_TICK / TICK_INTERVAL requests/sec peak.
--   Conservative on purpose; the "Get Link" family runs far hotter.
-- GAPFILL_SCAN_PER_TICK: pending IDs examined per tick in the gap-fill phase.
--   Pruning a non-real pending ID (legacy DoesItemExistByID flood) is local
--   and free, so we churn many per tick; only genuine re-requests count
--   against REQ_PER_TICK.
-- GAPFILL_MAX_STALL: patient window. If the pending count stops shrinking for
--   this many consecutive ticks (no item resolving), the remainder are treated
--   as permanently unresolvable — the server negative-caches "no such item",
--   so GetItemInfo returns nil and fires no event ever again — and are dropped
--   so the build converges to 0 gaps. Any real-but-slow item resets the stall
--   when it resolves, so reaching the limit means a full quiet window.
local WALK_PER_TICK        = 500
local REQ_PER_TICK         = 25
local TICK_INTERVAL        = 0.05
local GAPFILL_SCAN_PER_TICK = 2000
local GAPFILL_MAX_STALL    = 300  -- ~15s of zero progress at TICK_INTERVAL

-- Default ceiling for the ID walk. Generous on every flavour — non-existent
-- IDs cost only a local GetItemInfoInstant check (no request), so over-shooting
-- never wastes server traffic, and under-shooting would silently miss items
-- (incl. high-ID Era/SoD content). Editable per build via the tab; stored in SV.
local DEFAULT_TOP_ID = 240000

-- Field separator inside the packed per-item string. \31 (US) never appears
-- in an item name, so split is unambiguous.
local SEP = "\31"

-- ---------------------------------------------------------------------------
-- Store access
-- ---------------------------------------------------------------------------
local function Store()
    return addon.DB and addon.DB.global and addon.DB.global.itemDB
end

-- ---------------------------------------------------------------------------
-- Link reconstruction — from the stored id + quality + name, not GetItemInfo
-- (the item may not be cached at search time). A minimal `item:ID` body is
-- enough for GameTooltip:SetHyperlink and HandleModifiedItemClick (hover +
-- shift-click into chat) — the same interactive treatment RowList gives any
-- |Hitem:...|h cell text.
-- ---------------------------------------------------------------------------
local function BuildLink(id, quality, name)
    local r, g, b, hex = GetItemQualityColor(quality or 1)
    if not hex then
        hex = string.format("ff%02x%02x%02x",
            (r or 1) * 255, (g or 1) * 255, (b or 1) * 255)
    end
    return string.format("|c%s|Hitem:%d|h[%s]|h|r", hex, id, name)
end
ItemDB.BuildLink = function(_, id, quality, name) return BuildLink(id, quality, name) end

-- ---------------------------------------------------------------------------
-- Ingest one item ID. Returns (stored, didRequest):
--   stored      true if the item is now in the DB
--   didRequest  true if this call issued a NEW server request (counts against
--               the per-tick request budget)
-- GetItemInfoInstant is synchronous and returns the itemID (+ classID etc.)
-- for REAL items only — nil for non-existent IDs. So it doubles as the
-- existence gate AND lets us check the right class bucket immediately, making
-- re-walks (resume / late events / Rescan) idempotent and cheap.
-- ---------------------------------------------------------------------------
function ItemDB:_ingest(id)
    local store = Store()
    if not store then return false, false end

    local key = tostring(id)
    local instID, _, _, equipLoc, _, classID, subClassID = _GetItemInfoInstant(id)

    -- Non-existent ID — GetItemInfoInstant returned nothing. Never request it,
    -- and make sure it isn't lingering in `pending` (legacy builds, when we
    -- gated on DoesItemExistByID, flooded `pending` with these). This is the
    -- prune path that cleans an over-stuffed gap ledger.
    if not instID then
        if store.pending then store.pending[key] = nil end
        return false, false
    end

    classID    = classID    or -1
    subClassID = subClassID or -1

    local bucket = store.classes and store.classes[classID]
    if bucket and bucket[key] then
        if store.pending then store.pending[key] = nil end
        return true, false   -- already have it
    end

    local name, _, quality = _GetItemInfo(id)
    if name and name ~= "" then
        store.classes = store.classes or {}
        bucket = store.classes[classID]
        if not bucket then bucket = {}; store.classes[classID] = bucket end
        if not bucket[key] then store.count = (store.count or 0) + 1 end
        bucket[key] = name .. SEP .. (quality or 1) .. SEP .. subClassID .. SEP .. (equipLoc or "")
        if store.pending then store.pending[key] = nil end
        return true, false
    end

    -- Real item, not cached: the GetItemInfo call above just issued a server
    -- request. Record it as pending so a stop/resume (or a lost reply) can
    -- re-request it in the gap-fill phase rather than skipping it forever —
    -- the cursor has already advanced past this ID. The reply normally arrives
    -- via GET_ITEM_INFO_RECEIVED, which re-ingests and clears the pending mark.
    store.pending = store.pending or {}
    store.pending[key] = true
    return false, true
end

function ItemDB:_pendingCount()
    local store = Store()
    local n = 0
    if store and store.pending then
        for _ in pairs(store.pending) do n = n + 1 end
    end
    return n
end

-- ---------------------------------------------------------------------------
-- Progress callback — the tab assigns ItemDB.onProgress in :Draw and clears
-- it on release. Engine calls it after each tick / late arrival so the tab's
-- progress line stays live without rebuilding the table.
-- ---------------------------------------------------------------------------
function ItemDB:_fireProgress()
    if self.onProgress then self.onProgress() end
end

-- ---------------------------------------------------------------------------
-- Scan loop
-- ---------------------------------------------------------------------------
function ItemDB:_stopTicker()
    if self._ticker then self._ticker:Cancel(); self._ticker = nil end
end

function ItemDB:_tick()
    local store = Store()
    if not store then self:Pause(); return end
    local top = store.topID or DEFAULT_TOP_ID

    -- Phase 1 — forward walk from the persisted cursor. _ingest gates on
    -- GetItemInfoInstant internally, so non-existent IDs cost only that cheap
    -- local call and never issue a request. Advance up to WALK_PER_TICK IDs,
    -- but yield early once REQ_PER_TICK real requests have been issued.
    if (store.cursor or 1) <= top then
        local id = store.cursor or 1
        local walked, requested = 0, 0
        while id <= top and walked < WALK_PER_TICK and requested < REQ_PER_TICK do
            local _, didRequest = self:_ingest(id)
            if didRequest then requested = requested + 1 end
            id     = id + 1
            walked = walked + 1
        end
        store.cursor = id
        self:_fireProgress()
        return
    end

    -- Phase 2 — gap fill / prune. The walk is past topID; drain the pending
    -- set (real items whose reply hadn't arrived, plus any non-existent IDs a
    -- legacy DoesItemExistByID-gated build left behind). _ingest prunes the
    -- non-real ones for free and re-requests the real ones; replies arrive via
    -- GET_ITEM_INFO_RECEIVED. Examine up to GAPFILL_SCAN_PER_TICK IDs per tick
    -- but yield once REQ_PER_TICK real requests are issued, so pruning is fast
    -- while real requests stay throttled. Stop when pending empties or stalls.
    local pending = store.pending or {}
    local batch = {}
    for idStr in pairs(pending) do
        batch[#batch + 1] = idStr
        if #batch >= GAPFILL_SCAN_PER_TICK then break end
    end

    if #batch == 0 then
        self:_finalizeComplete()
        return
    end

    -- Prune every non-real ID in the batch for free (no request), and issue
    -- real requests only up to the per-tick budget. Crucially we do NOT break
    -- the loop when the request budget is hit — pruning the dead IDs that come
    -- after the first cluster of real items must still happen, or the count
    -- stalls with most of a legacy flood still present (which is exactly what
    -- happened: a break-on-budget left ~200k non-real IDs unpruned).
    local requested = 0
    for _, idStr in ipairs(batch) do
        local id = tonumber(idStr)
        if not _GetItemInfoInstant(id) then
            store.pending[idStr] = nil              -- non-real: prune (free)
        elseif requested < REQ_PER_TICK then
            local _, didRequest = self:_ingest(id)  -- store-from-cache or request
            if didRequest then requested = requested + 1 end
        end
        -- real but over the request budget this tick: leave it pending for
        -- a later tick (it'll be re-collected from `pending`).
    end

    -- Stall detection — if the pending count stops shrinking, some items
    -- just won't resolve; finalise rather than spin forever.
    local remaining = self:_pendingCount()
    if self._lastPending and remaining >= self._lastPending then
        self._stallTicks = (self._stallTicks or 0) + 1
    else
        self._stallTicks = 0
    end
    self._lastPending = remaining
    if self._stallTicks >= GAPFILL_MAX_STALL then
        -- A full patient window with zero progress: the remainder are
        -- permanently unresolvable (server negative-cached "no data", so
        -- GetItemInfo returns nil and fires no event — retrying can't produce
        -- a name, and a nameless item can't live in a name-search DB). Drop
        -- them so the build converges to 0 gaps. Anything real-but-slow would
        -- have reset the stall when it resolved, so we only reach here when
        -- nothing has come back for the whole window.
        if remaining > 0 then
            addon:Debug("ItemDB: gap-fill made no progress for " .. GAPFILL_MAX_STALL
                .. " ticks; dropping " .. remaining .. " unresolvable IDs to reach 0 gaps")
            store.pending = {}
        end
        self:_finalizeComplete()
        return
    end

    self:_fireProgress()
end

function ItemDB:_finalizeComplete()
    self:_stopTicker()
    self._running    = false
    self._stallTicks = 0
    self._lastPending = nil
    local s = Store()
    if s then
        s.complete = true
        s.builtAt  = GetServerTime()
    end
    self:_fireProgress()
end

-- Start (or resume) the walk from the saved cursor. No-op if already running
-- or if the required API is missing on this client.
function ItemDB:Start()
    if self._ticker then return end
    if not _GetItemInfoInstant then
        addon:Debug("ItemDB: GetItemInfoInstant unavailable on this client — cannot build")
        return
    end
    local store = Store()
    if not store then return end

    store.classes = store.classes or {}
    store.pending = store.pending or {}
    store.cursor  = store.cursor or 1
    store.topID   = store.topID or DEFAULT_TOP_ID

    -- Already fully built (walked to the end AND no pending gaps) — nothing to
    -- resume. Use Rebuild to redo from scratch.
    if store.complete and store.cursor > store.topID and self:_pendingCount() == 0 then
        addon:Debug("ItemDB: already complete — nothing to resume")
        return
    end

    -- Stamp locale/build on first build so the tab can warn when names are
    -- from a different locale or a stale client build.
    store.locale  = store.locale or GetLocale()
    store.build   = store.build  or select(4, GetBuildInfo())
    store.complete = false   -- starting/resuming means not (yet) complete

    self._running    = true
    self._stallTicks = 0
    self._lastPending = nil
    self._ticker     = C_Timer.NewTicker(TICK_INTERVAL, function() self:_tick() end)
    self:_fireProgress()
end

function ItemDB:Pause()
    self:_stopTicker()
    self._running = false
    -- cursor + pending live in SV, so the next Start (this session or after a
    -- logout) resumes the walk and still drains anything left in-flight.
    self:_fireProgress()
end

-- Re-walk the entire ID range from the start WITHOUT wiping existing data, to
-- pick up items missed on a previous run — e.g. an async reply that never
-- arrived, or that landed after gap-fill gave up (those linger in `pending`).
-- _ingest skips already-stored items cheaply (a bucket lookup, no request), so
-- only the genuine gaps cost a server request; `count` only grows for newly
-- stored items. classes / count / pending are all preserved.
function ItemDB:Rescan()
    self:Pause()
    local store = Store()
    if not store then return end
    store.cursor   = 1
    store.complete = false
    self:Start()
end

-- Scan ONLY the known gaps — the `pending` set (items we've confirmed exist
-- but don't yet have data for) — without re-walking the whole ID range. This
-- is the cheap "only fetch the ones we don't have" pass: it skips the ~240k
-- forward walk entirely and just re-requests the outstanding IDs, letting
-- GET_ITEM_INFO_RECEIVED fill them in. Use after a completed build to chase
-- stragglers; use Rescan instead if you suspect existing items that were
-- never walked at all (e.g. the walk never finished, or topID was raised).
function ItemDB:FillGaps()
    local store = Store()
    if not store then return end
    if self:_pendingCount() == 0 then
        addon:Debug("ItemDB: no gaps to fill")
        self:_fireProgress()
        return
    end
    self:Pause()
    -- Put the cursor past topID so _tick skips phase 1 (the walk) and goes
    -- straight to phase 2 (gap-fill on `pending`).
    store.cursor   = (store.topID or DEFAULT_TOP_ID) + 1
    store.complete = false
    self:Start()
end

-- Wipe everything and reset the cursor. Aliased as ClearAll for symmetry with
-- the log modules' clear API.
function ItemDB:Purge()
    self:Pause()
    local store = Store()
    if not store then return end
    store.classes  = {}
    store.pending  = {}
    store.count    = 0
    store.cursor   = 1
    store.complete = false
    store.builtAt  = nil
    self._stallTicks  = 0
    self._lastPending = nil
    self:_fireProgress()
end
ItemDB.ClearAll = ItemDB.Purge

-- ---------------------------------------------------------------------------
-- Read API
-- ---------------------------------------------------------------------------
function ItemDB:GetProgress()
    local store = Store()
    if not store then return {} end
    local cursor = store.cursor or 1
    local topID  = store.topID or DEFAULT_TOP_ID
    local pending = self:_pendingCount()

    -- phase: idle (paused, more to do) | walking | gapfill | complete
    local phase
    if store.complete then
        phase = "complete"
    elseif not self._running then
        phase = "idle"
    elseif cursor > topID then
        phase = "gapfill"
    else
        phase = "walking"
    end

    return {
        count     = store.count or 0,
        cursor    = cursor,
        topID     = topID,
        pending   = pending,
        complete  = store.complete or false,
        running   = self._running or false,
        phase     = phase,
        locale    = store.locale,
        build     = store.build,
        builtAt   = store.builtAt,
        localeNow = GetLocale(),
        buildNow  = select(4, GetBuildInfo()),
        available = _GetItemInfoInstant ~= nil,
    }
end

-- Classes present in the DB, with localized names + counts, sorted by name.
function ItemDB:GetClasses()
    local store = Store()
    local out = {}
    if not store or not store.classes then return out end
    for classID, bucket in pairs(store.classes) do
        local n = 0
        for _ in pairs(bucket) do n = n + 1 end
        local name = (_GetItemClassInfo and _GetItemClassInfo(classID)) or ("Class " .. classID)
        out[#out + 1] = { id = classID, name = name, count = n }
    end
    table.sort(out, function(a, b) return a.name < b.name end)
    return out
end

-- Subclasses present within a class, with localized names, sorted by name.
function ItemDB:GetSubClasses(classID)
    local store = Store()
    local out, seen = {}, {}
    local bucket = store and store.classes and store.classes[classID]
    if not bucket then return out end
    for _, packed in pairs(bucket) do
        local _, _, sub = strsplit(SEP, packed)
        sub = tonumber(sub) or -1
        if not seen[sub] then
            seen[sub] = true
            local name = (_GetItemSubClassInfo and _GetItemSubClassInfo(classID, sub))
                         or ("Subtype " .. sub)
            out[#out + 1] = { id = sub, name = name }
        end
    end
    table.sort(out, function(a, b) return a.name < b.name end)
    return out
end

-- Search the DB. opts:
--   query       substring match on name (case-insensitive)
--   classID     restrict to one class bucket (number) or nil for all
--   subClassID  restrict to one subclass (number) or nil
--   quality     restrict to one quality (number) or nil
--   max         result cap (default 300)
-- Returns an array of { id, name, quality, classID, subClassID, equipLoc,
-- typeName, subName, link }, sorted by name, plus the array carries a
-- `.capped` field when the cap was hit.
function ItemDB:Search(opts)
    opts = opts or {}
    local store = Store()
    if not store or not store.classes then return {} end

    local query    = (opts.query or ""):lower()
    local wantSub  = opts.subClassID
    local wantQual = opts.quality
    local max      = opts.max or 300

    local results = {}
    local capped  = false

    local function scanBucket(classID, bucket)
        local className = (_GetItemClassInfo and _GetItemClassInfo(classID)) or ("Class " .. classID)
        for idStr, packed in pairs(bucket) do
            local name, q, sub, eq = strsplit(SEP, packed)
            q   = tonumber(q)   or 1
            sub = tonumber(sub) or -1
            if (query == "" or name:lower():find(query, 1, true))
               and (not wantSub  or sub == wantSub)
               and (not wantQual or q == wantQual) then
                local id = tonumber(idStr)
                results[#results + 1] = {
                    id         = id,
                    name       = name,
                    quality    = q,
                    classID    = classID,
                    subClassID = sub,
                    equipLoc   = eq,
                    typeName   = className,
                    subName    = (_GetItemSubClassInfo and _GetItemSubClassInfo(classID, sub)) or "",
                    link       = BuildLink(id, q, name),
                }
                if #results >= max then capped = true; return true end
            end
        end
        return false
    end

    if opts.classID then
        local b = store.classes[opts.classID]
        if b then scanBucket(opts.classID, b) end
    else
        for classID, bucket in pairs(store.classes) do
            if scanBucket(classID, bucket) then break end
        end
    end

    table.sort(results, function(a, b) return a.name < b.name end)
    results.capped = capped
    return results
end

-- ---------------------------------------------------------------------------
-- Event wiring — capture asynchronous item-info replies while a scan (or its
-- drain window) is active. Gated on _running so there's zero overhead during
-- normal play.
-- ---------------------------------------------------------------------------
local _evt = CreateFrame("Frame")
_evt:RegisterEvent("GET_ITEM_INFO_RECEIVED")
_evt:SetScript("OnEvent", function(_, _, id, success)
    if not ItemDB._running then return end
    if not id then return end
    if success then
        -- Real data arrived — store it (clears the pending mark).
        ItemDB:_ingest(id)
    else
        -- The server confirmed it has NO data for this ID: a phantom item the
        -- client carries static stubs for (removed / test / cross-version)
        -- but that can never resolve to a name. Drop it from `pending` so it
        -- stops counting as a fillable gap and the build can reach 0.
        local store = Store()
        if store and store.pending then store.pending[tostring(id)] = nil end
    end
    ItemDB:_fireProgress()
end)

function ItemDB:OnEnable()
    -- Event wiring runs at file load; nothing to do on enable. The walk is
    -- user-triggered from the tab, never automatic.
end
