if not modules then modules = { } end modules ['data-tmp'] = {
    version   = 1.100,
    comment   = "companion to luat-lib.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

-- This module deals with caching data. It sets up the paths and implements loaders
-- and savers for tables. Best is to set the following variable. When not set, the
-- usual paths will be checked. Personally I prefer the (users) temporary path.
--
--   TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
--
-- Currently we do no locking when we write files. This is no real problem because
-- most caching involves fonts and the chance of them being written at the same time
-- is small. We also need to extend luatools with a recache feature.

local next, type = next, type
local pcall, loadfile, collectgarbage = pcall, loadfile, collectgarbage
local format, lower, gsub = string.format, string.lower, string.gsub
local concat, serialize, fastserialize, serializetofile = table.concat, table.serialize, table.fastserialize, table.tofile
local mkdirs, expanddirname, isdir, isfile = dir.mkdirs, dir.expandname, lfs.isdir, lfs.isfile
local is_writable, is_readable = file.is_writable, file.is_readable
local collapsepath, joinfile, addsuffix, dirname = file.collapsepath, file.join, file.addsuffix, file.dirname
local savedata = file.savedata
local formatters = string.formatters
local osexit, osdate, osuuid = os.exit, os.date, os.uuid
local removefile = os.remove
local md5hex = md5.hex

local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v end)
local trace_cache    = false  trackers.register("resolvers.cache",    function(v) trace_cache    = v end)

local report_caches    = logs.reporter("resolvers","caches")
local report_resolvers = logs.reporter("resolvers","caching")

local resolvers    = resolvers
local cleanpath    = resolvers.cleanpath
local resolvepath  = resolvers.resolve

local luautilities = utilities.lua

-- intermezzo

do

    local directive_cleanup = false  directives.register("system.compile.cleanup", function(v) directive_cleanup = v end)
    local directive_strip   = false  directives.register("system.compile.strip",   function(v) directive_strip   = v end)

    local compilelua = luautilities.compile

    function luautilities.compile(luafile,lucfile,cleanup,strip)
        if cleanup == nil then cleanup = directive_cleanup end
        if strip   == nil then strip   = directive_strip   end
        return compilelua(luafile,lucfile,cleanup,strip)
    end

end

-- end of intermezzo

caches              = caches or { }
local caches        = caches
local writable      = nil
local readables     = { }
local usedreadables = { }

local compilelua    = luautilities.compile
local luasuffixes   = luautilities.suffixes

caches.base         = caches.base or (LUATEXENGINE and LUATEXENGINE .. "-cache") or "luatex-cache"  -- can be local
caches.more         = caches.more or "context"       -- can be local
caches.defaults     = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }

local direct_cache  = false -- true is faster but may need huge amounts of memory
local fast_cache    = false
local cache_tree    = false

directives.register("system.caches.direct",function(v) direct_cache = true end)
directives.register("system.caches.fast",  function(v) fast_cache   = true end)

-- we could use a metatable for writable and readable but not yet

local function configfiles()
    return concat(resolvers.configurationfiles(),";")
end

local function hashed(tree)
    tree = gsub(tree,"[\\/]+$","")
    tree = lower(tree)
    local hash = md5hex(tree)
    if trace_cache or trace_locating then
        report_caches("hashing tree %a, hash %a",tree,hash)
    end
    return hash
end

local function treehash()
    local tree = configfiles()
    if not tree or tree == "" then
        return false
    else
        return hashed(tree)
    end
end

caches.hashed      = hashed
caches.treehash    = treehash
caches.configfiles = configfiles

local function identify()
    -- Combining the loops makes it messy. First we check the format cache path
    -- and when the last component is not present we try to create it.
    local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") -- forward ref
    if texmfcaches then
        for k=1,#texmfcaches do
            local cachepath = texmfcaches[k]
            if cachepath ~= "" then
                cachepath = resolvepath(cachepath)
                cachepath = cleanpath(cachepath)
                cachepath = collapsepath(cachepath)
                local valid = isdir(cachepath)
                if valid then
                    if is_readable(cachepath) then
                        readables[#readables+1] = cachepath
                        if not writable and is_writable(cachepath) then
                            writable = cachepath
                        end
                    end
                elseif not writable then
                    local cacheparent = dirname(cachepath)
                    if is_writable(cacheparent) then -- we go on anyway (needed for mojca's kind of paths)
                        mkdirs(cachepath)
                        if isdir(cachepath) and is_writable(cachepath) then
                            report_caches("path %a created",cachepath)
                            writable = cachepath
                            readables[#readables+1] = cachepath
                        end
                    end
                end
            end
        end
    end
    -- As a last resort we check some temporary paths but this time we don't
    -- create them.
    local texmfcaches = caches.defaults
    if texmfcaches then
        for k=1,#texmfcaches do
            local cachepath = texmfcaches[k]
            cachepath = resolvers.expansion(cachepath) -- was getenv
            if cachepath ~= "" then
                cachepath = resolvepath(cachepath)
                cachepath = cleanpath(cachepath)
                local valid = isdir(cachepath)
                if valid and is_readable(cachepath) then
                    if not writable and is_writable(cachepath) then
                        readables[#readables+1] = cachepath
                        writable = cachepath
                        break
                    end
                end
            end
        end
    end
    -- Some extra checking. If we have no writable or readable path then we simply
    -- quit.
    if not writable then
        report_caches("fatal error: there is no valid writable cache path defined")
        osexit()
    elseif #readables == 0 then
        report_caches("fatal error: there is no valid readable cache path defined")
        osexit()
    end
    -- why here
    writable = expanddirname(cleanpath(writable)) -- just in case
    -- moved here ( we have only one writable tree)
    local base = caches.base
    local more = caches.more
    local tree = cache_tree or treehash() -- we have only one writable tree
    if tree then
        cache_tree = tree
        writable = mkdirs(writable,base,more,tree)
        for i=1,#readables do
            readables[i] = joinfile(readables[i],base,more,tree)
        end
    else
        writable = mkdirs(writable,base,more)
        for i=1,#readables do
            readables[i] = joinfile(readables[i],base,more)
        end
    end
    -- end
    if trace_cache then
        for i=1,#readables do
            report_caches("using readable path %a (order %s)",readables[i],i)
        end
        report_caches("using writable path %a",writable)
    end
    identify = function()
        return writable, readables
    end
    return writable, readables
end

function caches.usedpaths(separator)
    local writable, readables = identify()
    if #readables > 1 then
        local result = { }
        local done = { }
        for i=1,#readables do
            local readable = readables[i]
            if readable == writable then
                done[readable] = true
                result[#result+1] = formatters["readable+writable: %a"](readable)
            elseif usedreadables[i] then
                done[readable] = true
                result[#result+1] = formatters["readable: %a"](readable)
            end
        end
        if not done[writable] then
            result[#result+1] = formatters["writable: %a"](writable)
        end
        return concat(result,separator or " | ")
    else
        return writable or "?"
    end
end

local r_cache = { }
local w_cache = { }

local function getreadablepaths(...) -- ... | tags
    local tags = { ... }
    local hash = concat(tags,"/")
    local done = r_cache[hash]
    if not done then
        local writable, readables = identify() -- exit if not found
        if #tags > 0 then
            done = { }
            for i=1,#readables do
                done[i] = joinfile(readables[i],...)
            end
        else
            done = readables
        end
        r_cache[hash] = done
    end
    return done
end

local function getwritablepath(...) -- ... | tags
    local tags = { ... }
    local hash = concat(tags,"/")
    local done = w_cache[hash]
    if not done then
        local writable, readables = identify() -- exit if not found
        if #tags > 0 then
            done = mkdirs(writable,...)
        else
            done = writable
        end
        w_cache[hash] = done
    end
    return done
end

local function setfirstwritablefile(filename,...)
    local wr = getwritablepath(...)
    local fullname = joinfile(wr,filename)
    return fullname, wr
end

local function setluanames(path,name)
    return
        format("%s/%s.%s",path,name,luasuffixes.tma),
        format("%s/%s.%s",path,name,luasuffixes.tmc)
end

local function getfirstreadablefile(filename,...)
    -- check if we have already written once
    local fullname, path = setfirstwritablefile(filename,...)
    if is_readable(fullname) then
        return fullname, path -- , true
    end
    -- otherwise search for pregenerated
    local rd = getreadablepaths(...)
    for i=1,#rd do
        local path = rd[i]
        local fullname = joinfile(path,filename)
        if is_readable(fullname) then
            usedreadables[i] = true
            return fullname, path -- , false
        end
    end
    -- else assume new written
    return fullname, path -- , true
end

caches.getreadablepaths     = getreadablepaths
caches.getwritablepath      = getwritablepath
caches.setfirstwritablefile = setfirstwritablefile
caches.getfirstreadablefile = getfirstreadablefile
caches.setluanames          = setluanames

-- -- not used:
--
-- function caches.define(category,subcategory)
--     return function()
--         return getwritablepath(category,subcategory)
--     end
-- end

-- This works best if the first writable is the first readable too. In practice
-- we can have these situations for file databases:
--
-- tma in readable
-- tma + tmb/c in readable
--
-- runtime files like fonts are written to the writable cache anyway

local checkmemory = utilities and utilities.lua and utilities.lua.checkmemory
local threshold   = 100 -- MB

function caches.loaddata(readables,name,writable)
    local used = checkmemory and checkmemory()
    if type(readables) == "string" then
        readables = { readables }
    end
    for i=1,#readables do
        local path   = readables[i]
        local loader = false
        local state  = false
        local tmaname, tmcname = setluanames(path,name)
        if isfile(tmcname) then
            state, loader = pcall(loadfile,tmcname)
        end
        if not loader and isfile(tmaname) then
            -- can be different paths when we read a file database from disk
            local tmacrap, tmcname = setluanames(writable,name)
            if isfile(tmcname) then
                state, loader = pcall(loadfile,tmcname)
            end
            compilelua(tmaname,tmcname)
            if isfile(tmcname) then
                state, loader = pcall(loadfile,tmcname)
            end
            if not loader then
                state, loader = pcall(loadfile,tmaname)
            end
        end
        if loader then
            loader = loader()
            if checkmemory then
                checkmemory(used,threshold)
            else -- generic
                collectgarbage("step") -- option, really slows down!
            end
            return loader
        end
    end
    return false
end

function caches.is_writable(filepath,filename)
    local tmaname, tmcname = setluanames(filepath,filename)
    return is_writable(tmaname)
end

local saveoptions = { compact = true, accurate = not JITSUPPORTED }

function caches.savedata(filepath,filename,data,fast)
    local tmaname, tmcname = setluanames(filepath,filename)
    data.cache_uuid = osuuid()
    if fast or fast_cache then
        savedata(tmaname,fastserialize(data,true))
    elseif direct_cache then
        savedata(tmaname,serialize(data,true,saveoptions))
    else
        serializetofile(tmaname,data,true,saveoptions)
    end
    compilelua(tmaname,tmcname)
end

-- moved from data-res:

local content_state = { }

function caches.contentstate()
    return content_state or { }
end

function caches.loadcontent(cachename,dataname,filename)
    if not filename then
        local name = hashed(cachename)
        local full, path = getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees")
        filename = joinfile(path,name)
    end
    local state, blob = pcall(loadfile,addsuffix(filename,luasuffixes.luc))
    if trace_cache and blob then
        report_caches("getting %s lua content from path %a","regular",filename)
    end
    if not blob then
        state, blob = pcall(loadfile,addsuffix(filename,luasuffixes.lua))
        if trace_cache and blob then
            report_caches("getting %s lua content from path %a","bytecode",filename)
        end
    end
    if blob then
        local data = blob()
        if data and data.content then
            if data.type == dataname then
                if data.version == resolvers.cacheversion then
                    local uuid = data.uuid
                    content_state[#content_state+1] = uuid
                    if trace_cache then
                        report_caches("registering content uuid %a for %a",uuid,filename)
                    end
                    if trace_locating then
                        report_resolvers("loading %a for %a from %a",dataname,cachename,filename)
                    end
                    return data.content
                else
                    report_resolvers("skipping %a for %a from %a (version mismatch)",dataname,cachename,filename)
                end
            else
                report_resolvers("skipping %a for %a from %a (datatype mismatch)",dataname,cachename,filename)
            end
        elseif trace_locating then
            report_resolvers("skipping %a for %a from %a (no content)",dataname,cachename,filename)
        end
    elseif trace_locating then
        report_resolvers("skipping %a for %a from %a (invalid file)",dataname,cachename,filename)
    end
end

function caches.collapsecontent(content)
    for k, v in next, content do
        if type(v) == "table" and #v == 1 then
            content[k] = v[1]
        end
    end
end

function caches.savecontent(cachename,dataname,content,filename)
    if not filename then
        local name = hashed(cachename)
        local full, path = setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees")
        filename = joinfile(path,name) -- is full
    end
    local luaname = addsuffix(filename,luasuffixes.lua)
    local lucname = addsuffix(filename,luasuffixes.luc)
    if trace_locating then
        report_resolvers("preparing %a for %a",dataname,cachename)
    end
    local uuid = osuuid()
    local data = {
        type    = dataname,
        root    = cachename,
        version = resolvers.cacheversion,
        date    = osdate("%Y-%m-%d"),
        time    = osdate("%H:%M:%S"),
        content = content,
        uuid    = uuid,
    }
    local ok = savedata(luaname,serialize(data,true))
    if trace_cache then
        report_caches("saving %a with uuid %a",luaname,uuid)
    end
    if ok then
        if trace_locating then
            report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname)
        end
        if compilelua(luaname,lucname) then
            if trace_locating then
                report_resolvers("%a compiled to %a",dataname,lucname)
            end
            return true
        else
            if trace_locating then
                report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname)
            end
            removefile(lucname)
        end
    elseif trace_locating then
        report_resolvers("unable to save %a in %a (access error)",dataname,luaname)
    end
end