// $Id: xml_report.cpp,v 1.17 2018/01/03 21:33:02 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Peeter Beerli, Mary Kuhner, Jon Yamato and Joseph Felsenstein */


#include <cassert>

#include "chainmanager.h"
#include "chainout.h"
#include "force.h"
#include "mathx.h"
#include "parameter.h"
#include "region.h"
#include "reportpage.h"
#include "timex.h"
#include "vector_constants.h"
#include "xml_report.h"

//------------------------------------------------------------------------------------

XMLReport::XMLReport(std::string filename)
    :   m_filename(filename),
        m_doc(filename.c_str())
{
}

XMLReport::~XMLReport()
{
}

void
XMLReport::AddSlicesTo( TiXmlElement * parentElem,
                        const Parameter& param,
                        force_type tag,
                        long regNo,
                        bool do_overall)
{

    bool isBayes = registry.GetChainParameters().IsBayesian();
    proftype ptype = param.GetProfileType();
    verbosity_type verbosity = registry.GetUserParameters().GetVerbosity();

    TiXmlElement * profileElem = new TiXmlElement("profile");
    std::string profTypeString = ToString(ptype);
    profileElem->SetAttribute("type",profTypeString.c_str());
    profileElem->SetAttribute("analysis", isBayes ? "bayesian" : "likelihood");
    parentElem->LinkEndChild(profileElem);

    double mleval = -DBL_BIG;
    double mleperc = -DBL_BIG;      // only for bayesian
    double llikeAtMle = -DBL_BIG;   // log likelihood for like, point prob for bayes

#if 0 // EWFIX.REMOVE
    vector<vector<centilepair> > centiles;
#endif

    vector<centilepair> CIvec;
    vector<centilepair> priorLikeVec;

    unsigned long paramindex = param.GetParamVecIndex();
    DoubleVec1d modifiers = registry.GetForceSummary().GetModifiers(paramindex);
    DoubleVec1d special_modifiers;  // used for fixed w/ growth

    if(do_overall)
    {
        mleval = param.GetOverallMLE();
        llikeAtMle = registry.GetForceSummary().GetOverallLlikeMle();
#if 0   // EWFIX.REMOVE
        centiles = param.GetOverallProfile();
#endif
        if(param.IsProfiled())
        {
            CIvec = param.GetOverallCIs();
            priorLikeVec = param.GetOverallPriorLikes();
        }
    }
    else
    {
        mleval = param.GetMLE(regNo);
        llikeAtMle = registry.GetForceSummary().GetLlikeMle(regNo);
#if 0   // EWFIX.REMOVE
        centiles = param.GetProfiles(regNo);
#endif
        if(param.IsProfiled())
        {
            CIvec = param.GetCIs(regNo);
            priorLikeVec = param.GetPriorLikes(regNo);
        }
    }

    // 999999999999999999999
    // EWFIX -- need to sort on double val
    std::map< std::pair<double, double>, TiXmlElement *> slices;

    if(param.IsProfiled())
    {
        if(isBayes)
        {
            mleperc = ReportPage::GetReverseCentile(CIvec,mleval);
            llikeAtMle = ReportPage::GetCentile(priorLikeVec,mleperc);
        }

        DoubleVec1d modTimesMleToPercs;
        if(ptype == profile_FIX )
        {
            if (param.IsForce(force_GROW) && verbosity != CONCISE && verbosity != NONE)
            {
                special_modifiers = vecconst::growthmultipliers;
                special_modifiers.insert(special_modifiers.end(),vecconst::growthfixed.begin(),vecconst::growthfixed.end());
            }
            if (isBayes)
            {
                if (param.IsForce(force_GROW) && verbosity != CONCISE && verbosity != NONE)
                {
                    modifiers = ProfPage::MakeGrowFixedModifiers(mleval);
                    for (unsigned long ind = 0; ind < modifiers.size(); ++ind)
                    {
                        modTimesMleToPercs.push_back(modifiers[ind]);
                    }
                }
                else
                {
                    for (unsigned long ind = 0; ind < modifiers.size(); ++ind)
                    {
                        modTimesMleToPercs.push_back(modifiers[ind]*mleval);
                    }
                }
                ProfPage::TradeValsForPercs(modTimesMleToPercs,CIvec);
            }
        }

        for (unsigned long ind = 0; ind < modifiers.size(); ++ind)
        {
            if (ptype == profile_NONE) continue;

            TiXmlElement * oneSlice = new TiXmlElement("profile-slice");

            double paramValue = -DBL_BIG;
            if(ptype == profile_FIX && (isBayes || verbosity == CONCISE || verbosity == NONE))
            {
                if (param.IsForce(force_GROW) && verbosity != CONCISE && verbosity != NONE)
                {
                    paramValue = modifiers[ind];
                }
                else
                {
                    paramValue = mleval * modifiers[ind];
                }
            }
            else
            {
                paramValue = ReportPage::GetCentile(CIvec,modifiers[ind]);
            }

            ///////////////////////////////////////////////////////////
            // <multiplier> -- only for fixed
            ///////////////////////////////////////////////////////////
            if(ptype == profile_FIX)
            {
                std::string modString = "multiplier";
                double modValue = modifiers[ind];

                if(verbosity != CONCISE && verbosity != NONE && param.IsForce(force_GROW))
                {
                    modValue = special_modifiers[ind];
                    if(paramValue == special_modifiers[ind])
                    {
                        modString = "fixed-value";
                    }
                }

                TiXmlElement * modElem = new TiXmlElement(modString);
                modElem->SetAttribute("value",ToString(modValue).c_str());
                oneSlice->LinkEndChild(modElem);
            }

            ///////////////////////////////////////////////////////////
            // <param-value>
            ///////////////////////////////////////////////////////////
            TiXmlElement * paramValueElem = new TiXmlElement("param-value");
            oneSlice->LinkEndChild(paramValueElem);
            paramValueElem->SetAttribute("value",ToString(paramValue).c_str());
            if(ptype == profile_PERCENTILE)
            {
                //If the profiler gave up before finding values with as low a
                // log likelihood as it would have liked, that information was
                // saved in the analyzer, and we can get it out here to tell
                // users that the value is "<1.53" instead of "1.53".
                if (param.CentileIsExtremeLow(modifiers[ind], regNo))
                {
                    if (param.CentileIsExtremeHigh(modifiers[ind], regNo))
                    {
                        // EWFIX -- do we have better warn status?
                        paramValueElem->SetAttribute("warning","failed");
                        paramValueElem->SetAttribute("value","***");
                    }
                    else
                    {
                        paramValueElem->SetAttribute("warning","lower");
                    }
                }
                else if (param.CentileIsExtremeHigh(modifiers[ind], regNo))
                {
                    paramValueElem->SetAttribute("warning","higher");
                }
                else if (param.CentileHadWarning(modifiers[ind], regNo))
                {
                    paramValueElem->SetAttribute("warning","other");
                }
            }

            ///////////////////////////////////////////////////////////
            // <percentile>
            ///////////////////////////////////////////////////////////
            if(ptype == profile_PERCENTILE)
            {
                TiXmlElement * percElem = new TiXmlElement("percentile");
                percElem->SetAttribute("value",ToString(modifiers[ind]));
                oneSlice->LinkEndChild(percElem);
            }
            if(ptype == profile_FIX && isBayes)
            {
                TiXmlElement * percElem = new TiXmlElement("percentile");

                double modVal = modifiers[ind];
                if (! (param.IsForce(force_GROW) && verbosity != CONCISE && verbosity != NONE))
                {
                    modVal *= mleval;
                }

                double val = ReportPage::GetReverseCentile(CIvec,modVal);
                percElem->SetAttribute("value",ToString(val));
                oneSlice->LinkEndChild(percElem);
            }

            ///////////////////////////////////////////////////////////
            // <log-likelihood> -- not for bayes
            ///////////////////////////////////////////////////////////
            if(!isBayes)
            {
                TiXmlElement * logLike = new TiXmlElement("log-likelihood");
                double lnl = ReportPage::GetCentile(priorLikeVec,modifiers[ind]);

#if 0
                // EWFIX.HACK.HACK.HACK -- I don't understand what's going
                // on in reportpage.cpp, but at least this gets the right
                // values out
                if(lnl == FLAGLONG && ptype == profile_FIX && param.IsForce(force_GROW) && verbosity != CONCISE && verbosity != NONE)
                {
                    lnl = ReportPage::GetCentile(priorLikeVec,modifiers[ind]/mleval);
                }
#endif

                logLike->SetAttribute("value",ToString(lnl).c_str());
                oneSlice->LinkEndChild(logLike);
            }

            ///////////////////////////////////////////////////////////
            // <point-probability> -- only for bayes
            ///////////////////////////////////////////////////////////
            if(isBayes)
            {
                TiXmlElement * pointProb = new TiXmlElement("point-probability");
                double lnl = -DBL_BIG;
                if(ptype == profile_FIX)
                {
                    lnl = ReportPage::GetCentile(priorLikeVec,modTimesMleToPercs[ind]);
                }
                else
                {
                    lnl = ReportPage::GetCentile(priorLikeVec,modifiers[ind]);
                }
                std::string likeStr = (lnl == -DBL_BIG) ? "***" : ToString(lnl);
                pointProb->SetAttribute("value",likeStr.c_str());
                oneSlice->LinkEndChild(pointProb);
            }

            slices[std::pair<double, double>(paramValue,modifiers[ind])]=oneSlice;

        }
    }

    // add in element at MLE/MPE
    bool haveMle = false;

    // element may already be there, but not marked as MLE
    // I believe this only happens for profile_FIX && isBayes
    // It can also be there because we've hit the maximum or
    // minimum value -- but in that case we want the duplicate
    // entries
    std::map< std::pair<double, double>, TiXmlElement *>::const_iterator sliter;
    for(sliter = slices.begin(); sliter != slices.end(); sliter++)
    {
        std::pair<double, double> sortVal = (*sliter).first;
        double paramValue = sortVal.first;
        if(isBayes && ptype == profile_FIX && CloseEnough(paramValue,mleval))
        {
            haveMle = true;
            assert(sortVal.second == 1.0 || param.IsForce(force_GROW));
            ((*sliter).second)->SetAttribute("special", "mpe");
        }
    }

    if (!haveMle)
    {
        TiXmlElement * newSlice = new TiXmlElement("profile-slice");
        double secondSort = ptype == profile_FIX ? 1.0 : 0.5;
        slices[std::pair<double, double>(mleval,secondSort)] = newSlice;

        newSlice->SetAttribute("special", isBayes ? "mpe" : "mle");

        if(ptype == profile_FIX)
        {
            TiXmlElement * multiplierElem = new TiXmlElement("multiplier");
            newSlice->LinkEndChild(multiplierElem);
            multiplierElem->SetAttribute("value",ToString(1.0).c_str());
        }

        TiXmlElement * paramValueElem = new TiXmlElement("param-value");
        newSlice->LinkEndChild(paramValueElem);
        paramValueElem->SetAttribute("value",ToString(mleval).c_str());

        if(param.IsProfiled())
        {
            if(isBayes)
            {
                TiXmlElement * percElem = new TiXmlElement("percentile");
                newSlice->LinkEndChild(percElem);
                percElem->SetAttribute("value",ToString(mleperc).c_str());

                // EWFIX -- wrong for bayes/fix/grow ?? test this !!
                TiXmlElement * pointProbElem = new TiXmlElement("point-probability");
                newSlice->LinkEndChild(pointProbElem);
                pointProbElem->SetAttribute("value",ToString(llikeAtMle).c_str());
            }
            else
            {
                TiXmlElement * logLikeElem = new TiXmlElement("log-likelihood");
                newSlice->LinkEndChild(logLikeElem);
                logLikeElem->SetAttribute("value",ToString(llikeAtMle).c_str());
            }
        }

    }

    for(sliter = slices.begin(); sliter != slices.end(); sliter++)
    {
        profileElem->LinkEndChild((*sliter).second);
    }
}

TiXmlElement *
XMLReport::MakeParameter(const Parameter& param, force_type force_tag)
{

    string paramname = param.GetName();
    ReportPage::TrimString(paramname);
    string shortparamname = param.GetShortName();
    ReportPage::TrimString(shortparamname);
    unsigned long pindex = param.GetParamVecIndex();
    proftype ptype = param.GetProfileType();

    TiXmlElement * paramElem = new TiXmlElement("parameter");
    paramElem->SetAttribute("short-name",shortparamname.c_str());
    paramElem->SetAttribute("name",paramname.c_str());
    paramElem->SetAttribute("index",ToString(pindex));
    paramElem->SetAttribute("type",ToString(ptype));

    long nregs = registry.GetDataPack().GetNRegions();
    for (long reg = 0; reg <= nregs; ++reg)
    {
        if (reg == nregs && nregs <= 1) break;
        bool do_overall = (reg==nregs && nregs > 1);

        TiXmlElement * estimateElem = new TiXmlElement("estimate");
        paramElem->LinkEndChild(estimateElem);
        if(do_overall)
        {
            estimateElem->SetAttribute("type","overall");
        }
        else
        {
            estimateElem->SetAttribute("type","single-region");
            std::string regionname = registry.GetDataPack().GetRegion(reg).GetRegionName();
            estimateElem->SetAttribute("region-name",regionname.c_str());
        }

        AddSlicesTo(estimateElem,param,force_tag,reg,do_overall);
    }
    return paramElem;
}

TiXmlElement *
XMLReport::MakeParameters()
{

    TiXmlElement * parametersElem = new TiXmlElement("parameters");

    const ForceVec forces = registry.GetForceSummary().GetAllForces();

    ForceVec::const_iterator fit;
    for(fit = forces.begin(); fit != forces.end(); ++fit)
    {
        // organizing parameters by force
        TiXmlElement * forceElem = new TiXmlElement("force");
        parametersElem->LinkEndChild(forceElem);

        // identify the force
        string forcename = (*fit)->GetFullparamname();
        string shortforcename = (*fit)->GetShortparamname();
        force_type tag = (*fit)->GetTag();

        forceElem->SetAttribute("long-name",forcename.c_str());
        forceElem->SetAttribute("short-name",shortforcename.c_str());

        const vector<Parameter>& parameters = (*fit)->GetParameters();
        vector<Parameter>::const_iterator param;
        for (param = parameters.begin(); param != parameters.end(); ++param)
        {
            ParamStatus mystatus = param->GetStatus();
            if (!mystatus.Valid()) continue;

            TiXmlElement * paramElem = MakeParameter(*param,tag);
            forceElem->LinkEndChild(paramElem);
        }
    }
    // RegionGammaInfo should go here
    return parametersElem;
}

TiXmlElement *
XMLReport::MakeChainReport(const ChainOut& co,size_t regNo, size_t repNo, size_t chainNo)
{
    TiXmlElement * elem = new TiXmlElement("chain-info");
    elem->SetAttribute("region",ToString(regNo).c_str());
    elem->SetAttribute("replicate",ToString(repNo).c_str());
    elem->SetAttribute("chain",ToString(chainNo).c_str());

    // info on timing
    TiXmlElement * timeElem = new TiXmlElement("runtime");
    time_t starttime = co.GetStarttime();
    time_t endtime = co.GetEndtime();
    timeElem->SetAttribute("start",PrintTime(starttime).c_str());
    timeElem->SetAttribute("end",PrintTime(endtime).c_str());
    timeElem->SetAttribute("seconds",ToString(endtime-starttime).c_str());
    elem->LinkEndChild(timeElem);

    // info on discarded trees
    TiXmlElement * discardsElem = new TiXmlElement("discarded-trees");

    TiXmlElement * badTreesElem = new TiXmlElement("bad-trees");
    badTreesElem->SetAttribute("value",ToString(co.GetNumBadTrees()).c_str());
    discardsElem->LinkEndChild(badTreesElem);

    TiXmlElement * tinyPopTreesElem = new TiXmlElement("tiny-pop-trees");
    tinyPopTreesElem->SetAttribute("value",ToString(co.GetTinyPopTrees()).c_str());
    discardsElem->LinkEndChild(tinyPopTreesElem);

    TiXmlElement * zeroDlTreesElem = new TiXmlElement("zero-dl-trees");
    zeroDlTreesElem->SetAttribute("value",ToString(co.GetZeroDLTrees()).c_str());
    discardsElem->LinkEndChild(zeroDlTreesElem);

    TiXmlElement * longBranchTreesElem = new TiXmlElement("long-branch-trees");
    longBranchTreesElem->SetAttribute("value",ToString(co.GetStretchedTrees()).c_str());
    discardsElem->LinkEndChild(longBranchTreesElem);

    elem->LinkEndChild(discardsElem);

    // info on acceptance rates
    TiXmlElement * acceptRatesElem = new TiXmlElement("acceptance-rates");
    elem->LinkEndChild(acceptRatesElem);

    TiXmlElement * overallAcceptRate = new TiXmlElement("overall-acceptance");
    overallAcceptRate->SetAttribute("value",ToString(co.GetAccrate()).c_str());
    acceptRatesElem->LinkEndChild(overallAcceptRate);

    ratemap::const_iterator iter;
    ratemap rates = co.GetAllAccrates();
    for(iter=rates.begin(); iter != rates.end(); iter++)
    {
        std::string rateType = (*iter).first;
        std::pair<long,long> values = (*iter).second;
        long accepted = values.first;
        long proposed = values.second;

        TiXmlElement * rateElem = new TiXmlElement("arranger-rate");
        rateElem->SetAttribute("type",rateType);
        rateElem->SetAttribute("accepted",ToString(accepted).c_str());
        rateElem->SetAttribute("proposed",ToString(proposed).c_str());
        acceptRatesElem->LinkEndChild(rateElem);
    }

    // unique sampled values (for Bayesian)
    vector<long> bayesunique = co.GetBayesUnique();
    if (!bayesunique.empty())
    {
        TiXmlElement * uniqBayesElem = new TiXmlElement("unique-sampled-values");
        elem->LinkEndChild(uniqBayesElem);

        const ParamVector paramvec(true);
        assert(paramvec.size() == bayesunique.size());
        for (unsigned long pnum=0; pnum<paramvec.size(); pnum++)
        {
            ParamStatus mystatus = paramvec[pnum].GetStatus();
            if (mystatus.Inferred())
            {
                TiXmlElement * pElem = new TiXmlElement("parameter");
                uniqBayesElem->LinkEndChild(pElem);

                pElem->SetAttribute("name",paramvec[pnum].GetName());
                pElem->SetAttribute("samples",bayesunique[pnum]);

                if (bayesunique[pnum] < 50)
                {
                    pElem->SetAttribute("warning","too few");
                }
            }
        }
    }

    return elem;
}

TiXmlElement *
XMLReport::MakeWarnings()
{
    TiXmlElement * warnElem = new TiXmlElement("warnings");

    TiXmlElement * frontEndElem = new TiXmlElement("front-end-warnings");
    warnElem->LinkEndChild(frontEndElem);
    TiXmlComment * warnComment = new TiXmlComment();
    warnComment->SetValue("foo!");
    warnElem->LinkEndChild(warnComment);

    TiXmlElement * backEndElem = new TiXmlElement("back-end-warnings");
    warnElem->LinkEndChild(backEndElem);
    warnComment = new TiXmlComment();
    warnComment->SetValue("fie!");
    warnElem->LinkEndChild(warnComment);

    return warnElem;
}

void
XMLReport::AddRangeElements(TiXmlElement * traitElem,rangeset sites,std::string label)
{
    TiXmlElement * range = new TiXmlElement("range");
    traitElem->LinkEndChild(range);
    range->SetAttribute("type",label.c_str());

    for(rangeset::iterator i = sites.begin(); i != sites.end(); i++)
    {
        rangepair rp = *i;
        TiXmlElement * sites = new TiXmlElement("sites");
        range->LinkEndChild(sites);
        sites->SetAttribute("start",ToString(rp.first).c_str());
        sites->SetAttribute("stop",ToString(rp.second-1).c_str());
    }
}

TiXmlElement *
XMLReport::Write(const ChainManager& chainMan)
{
    TiXmlElement * topItem = new TiXmlElement("lamarc-run-report");
    m_doc.LinkEndChild( topItem );

    // parameters
    TiXmlElement * parameters = MakeParameters();
    topItem->LinkEndChild(parameters);

    // mapping
    if (registry.GetDataPack().AnyMapping())
    {
        TiXmlElement * traitsElem = new TiXmlElement("traits");
        topItem->LinkEndChild(traitsElem);

        for (long reg=0; reg<registry.GetDataPack().GetNRegions(); reg++)
        {
            const Region& region = registry.GetDataPack().GetRegion(reg);
            for (long mloc=0; mloc<region.GetNumMovingLoci(); mloc++)
            {
                const Locus& locus = region.GetMovingLocus(mloc);


                TiXmlElement * traitElem = new TiXmlElement("trait");
                traitsElem->LinkEndChild(traitElem);

                traitElem->SetAttribute("name",locus.GetName().c_str());
                traitElem->SetAttribute("type",ToString(locus.GetAnalysisType()).c_str());


                long regoffset = region.GetSiteSpan().first;
                std::set<std::pair<double, long int> > orderedsites = locus.MakeOrderedSites(regoffset);

                rangeset bestsites = locus.GetBestSites(orderedsites);
                AddRangeElements(traitElem,bestsites,"best");

                rangeset top5 = locus.GetTopSites(orderedsites,0.05);
                AddRangeElements(traitElem,top5,"0.05");

                rangeset top50 = locus.GetTopSites(orderedsites,0.50);
                AddRangeElements(traitElem,top50,"0.50");

                rangeset top95 = locus.GetTopSites(orderedsites,0.95);
                AddRangeElements(traitElem,top95,"0.95");
            }
        }
    }

    // chains and arrangers report
    TiXmlElement * chainPackElem = new TiXmlElement("chains");
    topItem->LinkEndChild(chainPackElem);
    vector<vector<vector<ChainOut > > > chains = chainMan.GetChainPack().GetAllChains();
    for(size_t i = 0; i < chains.size(); i++)
    {
        for(size_t j = 0; j < chains[i].size(); j++)
        {
            for(size_t k = 0; k < chains[i][j].size(); k++)
            {
                const ChainOut & co = chains[i][j][k];
                chainPackElem->LinkEndChild(MakeChainReport(co,i,j,k));
            }
        }
    }

    // EWFIX.ADD full paths to output files ??

    // EWFIX.ADD warnings given
#if 0
    TiXmlElement * warnings = MakeWarnings();
    topItem->LinkEndChild(warnings);
#endif

    // this writes it all out -- file name already set at construction
    // of m_doc
    m_doc.SaveFile();

    return topItem;
}

//____________________________________________________________________________________
