// $Id: arranger.cpp,v 1.123 2018/01/03 21:33:03 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Peter Beerli, Mary Kuhner, Jon Yamato and Joseph Felsenstein */


#include <cassert>

#include "local_build.h"

#include "arranger.h"
#include "event.h"
#include "tree.h"
#include "force.h"
#include "forceparam.h"
#include "forcesummary.h"
#include "random.h"
#include "errhandling.h"                // Arranger can throw data_error
#include "mathx.h"
#include "registry.h"
#include "likelihood.h"                 // for Bayesian arranger functions
#include "chainstate.h"
#include "timemanager.h"                // used to handle all rearrangements of of stick/stair
#include "plforces.h"                   // used in the Brownian-Bridge for stick/stair rearrangement
#include "fc_status.h"                  // for tracking final coalescence in the ResimArranger's
                                        // functions, Resimulate() and DropAll()

#ifdef DMALLOC_FUNC_CHECK
#include "/usr/local/include/dmalloc.h"
#endif

using namespace std;

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger::Arranger(double timing)
    :  m_timing(timing),
       m_savetiming(timing),
       m_tree(NULL),
       randomSource(&(registry.GetRandom()))
{
    // intentionally blank
} // Arranger::Arranger

//------------------------------------------------------------------------------------

Arranger::Arranger(const Arranger& src)
    : m_timing(src.m_timing),
      m_savetiming(src.m_savetiming),
      randomSource(src.randomSource)
{
    m_tree = NULL;
} // Arranger::Arranger

//------------------------------------------------------------------------------------

void Arranger::SetTiming(double t)
{
    if(t < 0.0)
    {
        data_error e("arranger timing cannot be < 0.0");
        throw e;
    }

    // We used to barf for t > 1.0, but now we normalize the timings later, so we don't do that.

    m_timing = t;
} // Arranger::SetTiming

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

ResimArranger::ResimArranger(const ForceSummary & fs, double timing)
    : Arranger(timing),
      m_eventvec(fs.CreateEventVec()),
      m_hasLogisticSelection(false),    // set in SetParameters
      m_logsel_cutoff(FLAGDOUBLE),      // ditto
      m_activelist(registry.GetDataPack()),
      m_inactivelist(registry.GetDataPack())
{
    // inform each Event of the Arranger for callbacks
    vector<Event*>::iterator event = m_eventvec.begin();
    vector<Event*>::iterator eventend = m_eventvec.begin();
    for ( ; event != eventend; ++event)
    {
        (*event)->SetArranger(*this);
    }
} // ResimArranger ctor

//------------------------------------------------------------------------------------

ResimArranger::ResimArranger(const ResimArranger& src)
    : Arranger(src),
      m_eventvec(),
      m_hasLogisticSelection(false),    // set in SetParameters
      m_logsel_cutoff(FLAGDOUBLE),      // ditto
      m_activelist(registry.GetDataPack()),
      m_inactivelist(registry.GetDataPack())
{
    vector<Event*>::const_iterator event = src.m_eventvec.begin();
    vector<Event*>::const_iterator end = src.m_eventvec.end();
    Event* newevent;
    for( ; event != end; ++event)
    {
        newevent = (*event)->Clone();
        newevent->SetArranger(*this);
        m_eventvec.push_back(newevent);
    }

    // We do not copy m_xactives etc. because they are only scratchpads for Event use.
} // ResimArranger::ResimArranger

//------------------------------------------------------------------------------------

ResimArranger::~ResimArranger()
{
    ClearEventVec();
} // ResimArranger::~ResimArranger

//------------------------------------------------------------------------------------

void ResimArranger::ClearEventVec()
{
    vector<Event*>::iterator it = m_eventvec.begin();
    vector<Event*>::iterator end = m_eventvec.end();
    for ( ; it != end; ++it)
    {
        delete *it;
    }
    m_eventvec.clear();

} // ClearEventVec

//------------------------------------------------------------------------------------

void ResimArranger::SetParameters(ChainState & chainstate)
{
    assert(m_eventvec.size() != 0);     // *Some* events must be possible!

    vector<Event*>::iterator it = m_eventvec.begin();
    vector<Event*>::iterator end = m_eventvec.end();

    m_hasLogisticSelection = false;

    ForceParameters starts(chainstate.GetParameters());
    // MDEBUG not happy with presence of force-specific overflow code here in a general use function.
    for ( ; it != end; ++it)
    {
        (*it)->InstallParameters(starts);
        if (registry.GetForceSummary().HasLogisticSelection())
        {
            // Compute and store the maximum starttime, beyond which over/underlow will ensue.
            double theta_A0 = starts.GetRegionalThetas()[0];
            double theta_a0 = starts.GetRegionalThetas()[1];
            double s = starts.GetLogisticSelectionCoefficient()[0];
            if (theta_A0 <= 0.0 || theta_a0 <= 0.0)
                throw impossible_error("Invalid Theta received by ResimArranger::SetParameters().");
            m_hasLogisticSelection = true;
            if (s > 0.0)
                m_logsel_cutoff = (EXPMAX - log(theta_a0) - 4.0 * LOG10) / s;
            else if (s < 0.0)
                m_logsel_cutoff = (EXPMAX - log(theta_A0) - 4.0 * LOG10) / (-s);
            else
                m_logsel_cutoff = DBL_MAX; // no cutoff
        }
    }

} // SetParameters

//------------------------------------------------------------------------------------

double ResimArranger::EventTime(Event*& returnevent, double eventT, double maxtime)
{
    // NB "maxtime" is the time at the start of the next interval.  Each Event is responsible
    // for returning a time that respects that boundary; the Event itself knows whether it must
    // avoid tying the boundary or only passing it.
    m_xactives = m_activelist.GetBranchXParts();
    m_xinactives = m_inactivelist.GetBranchXParts();

    m_pactives = m_activelist.GetBranchParts();
    m_pinactives = m_inactivelist.GetBranchParts();

    double time = 0.0;

    vector<Event*>::iterator event;
    map<double, Event *> times;

    for (event = m_eventvec.begin(); event != m_eventvec.end(); ++event)
    {
        time = (*event)->PickTime(eventT, maxtime);
        if (time != FLAGDOUBLE)
        {
            assert(time >= 0.0);        // Should not be negative, ever!
            times.insert(make_pair(time, *event));
        }
    }

    // The first element in the map is now the smallest, since maps are intrinsically sorted.

    returnevent = NULL;
    // No winning horse exists.
    if (times.empty()) return FLAGLONG;

    map<double, Event*>::const_iterator mapit = times.begin();
    double newT(mapit->first);

    // Winning horse is at the interval end and is not a ties-allowed horse.
    Event* winner = (*mapit).second;
#if 0
    if (newT == eventT && !(winner->TiesAllowed())) return FLAGLONG;
#endif

    // Winning horse returned DBL_BIG.
    if (newT >= DBL_BIG)
    {
        string msg("maximal length branch won the horse race in");
        msg += " ResimArranger::EventTime";
        stretched_error e(msg);
        throw e;
    }

    // Okay, we have a valid winner!
    if (m_hasLogisticSelection && eventT + newT > m_logsel_cutoff)
    {
        // Shrink this new interval a bit, to prevent over/underflow.  The cutoff is computed in SetParameters().
        // It's a simple function of the current (driving) values of theta_A0, theta_a0, and s.
        newT = m_logsel_cutoff - eventT;
    }
    returnevent = winner;
    return newT;
} // ResimArranger::EventTime

//------------------------------------------------------------------------------------

void ResimArranger::DropAll(double eventT)
{
    double time;
    Event* eventptr = NULL;

    m_inactivelist.Clear();

    FC_Status fcstatus;
    m_activelist.IncreaseFCCount(fcstatus);

    while (m_activelist.Size() > 1)
    {
        time = EventTime(eventptr, eventT, DBL_MAX);
        assert (time != FLAGLONG);
        assert (eventptr != NULL);
        double nextT = time;
        eventptr->DoEvent(nextT, fcstatus);
        eventT = nextT;
    }

    // Remove the root from the m_activelist then attach it to the tree.
    m_tree->AttachBase(m_activelist.RemoveFirst());
} // DropAll

//------------------------------------------------------------------------------------

double ResimArranger::Activate(Tree * oldtree)
{
    // cerr << endl << "in ResimArranger::Activate calling RevalidateRange" << endl;
    assert(m_tree->GetTimeList().RevalidateAllRanges());
    Branch_ptr activebranch = m_tree->ActivateBranch(oldtree);
    m_activelist.Append(activebranch);
    return activebranch->m_eventTime;
} // Activate

//------------------------------------------------------------------------------------

void ResimArranger::Resimulate(double eventT, ChainState & chainstate)
{
    double nextT, time;
    Event* eventptr;

    double rootT = m_tree->RootTime();

    vector<Branch_ptr> newinactives = m_tree->FirstInterval(eventT);
    unsigned long i;
    m_inactivelist.Clear();

    for (i = 0; i < newinactives.size(); ++i)
    {
        m_inactivelist.Collate(newinactives[i]);
    }
    nextT = m_inactivelist.IntervalBottom();

    FC_Status fcstatus;
    m_activelist.IncreaseFCCount(fcstatus);
    m_inactivelist.IncreaseFCCount(fcstatus);

    // The following loop resimulates lineages down the tree.  It will terminate when the
    // Event objects agree that no more events are possible, or when the root is reached,
    // at which point DropAll is invoked.
    while (true)
    {
        // Poll the Event objects to see if we're done yet.

        if (StopNow()) return;

        time = EventTime(eventptr, eventT, nextT);

        // If an event is possible we carry it out, then return to the top of the loop while
        // remaining in the same interval, since further events may occur.
        if (time != FLAGLONG)
        {
            eventT = time;
            eventptr->DoEvent(eventT, fcstatus);
            continue;
        }

        // if we are at the root, finish up using DropAll()
        if (nextT == rootT)
        {
            m_activelist.Append(m_tree->ActivateRoot(fcstatus));
            DropAll(rootT);
            return;
        }

        // otherwise go on to the next interval
        eventT = nextT;
        nextT = NextInterval(eventT, fcstatus);
    }
} // Resimulate

//------------------------------------------------------------------------------------

double ResimArranger::NextInterval(double lastT, FC_Status & fcstatus)
{
    // Remove first branch from inactive list.
    Branch_ptr pBranch = m_inactivelist.RemoveFirst();

    // Insert its parent into inactive list.
    Branch_ptr pParent = pBranch->Parent(0);
    m_inactivelist.Collate(pParent);

    if (pParent->Child(1))              // If the branch has a sibling,
    {
        m_inactivelist.RemoveFirst();   // remove it from the inactive list also.
        rangeset coalesced_sites = Intersection(pParent->Child(0)->GetLiveSites(), pParent->Child(1)->GetLiveSites());
#if FINAL_COALESCENCE_ON
        fcstatus.Decrement_FC_Counts(coalesced_sites);
#endif
    }
    else if (pBranch->Parent(1))
    {
        // If branch has a second parent, insert it.
        m_inactivelist.Collate(pBranch->Parent(1));
#if FINAL_COALESCENCE_ON
        pBranch->Parent(1)->UpdateBranchRange(fcstatus.Coalesced_Sites(), true);
#else
        rangeset emptyset;
        pBranch->Parent(1)->UpdateBranchRange(emptyset, false);
#endif
    }
#if FINAL_COALESCENCE_ON
    pParent->UpdateBranchRange(fcstatus.Coalesced_Sites(), true);
#else
    rangeset emptyset;
    pParent->UpdateBranchRange(emptyset, false);
#endif

    // MARYDEBUG--kludge til think of something better used to handle rectree updating --
    // should probably move rectree updating into arranger?
    m_tree->NextInterval(pBranch);

    // return the time at the bottom of the current interval.
    return m_inactivelist.IntervalBottom();
} // NextInterval

//------------------------------------------------------------------------------------

void ResimArranger::CleanupAfterResimulate()
{
    m_tree->Prune();
} // CleanupAfterResimulate

//------------------------------------------------------------------------------------

bool ResimArranger::StopNow() const
{
    // Poll the Events to see if they are Done().  If any are not, return false.
    vector<Event*>::const_iterator event = m_eventvec.begin();
    vector<Event*>::const_iterator end = m_eventvec.end();

    for ( ; event != end; ++event)
    {
        if (!(*event)->Done()) return false;
    }

    return true;
} // StopNow

//------------------------------------------------------------------------------------

void ResimArranger::Rearrange(ChainState & chstate)
{
    // This is a "template method" giving the steps of rearrangement.  It may throw
    // a "rejecttree_error" to signal a newtree that should be discarded.
    m_activelist.Clear();               // m_activelist "clear" happens here instead of
    // in Activate in case we want to "Activate" more than one branch for a given "drop".
    m_tree = chstate.GetTree();
    Tree * oldtree = chstate.GetOldTree();
    double eventT = Activate(oldtree);
    Resimulate(eventT, chstate);
    CleanupAfterResimulate();
    assert(m_tree->IsValidTree());
} // Rearrange

//------------------------------------------------------------------------------------

void ResimArranger::ScoreRearrangement(ChainState & chstate)
{
    // Calculate data likelihood of new tree (stores it in the Tree object).
    chstate.GetTree()->CalculateDataLikes();
} // ScoreRearrangement

//------------------------------------------------------------------------------------

bool ResimArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    // NB:  We assume that the tips of the tree were not changed, and thus we only accept/reject the body of the tree.
    if (badtree)                        // reject this tree immediately
    {
        chstate.OverwriteTree();
        return false;
    }

    Tree * tree = chstate.GetTree();
    Tree * oldtree = chstate.GetOldTree();

    double test = (tree->GetDLValue() - oldtree->GetDLValue()) / temperature;
    test += Hastings(chstate);

#ifndef LAMARC_QA_SINGLE_DENOVOS        // Make LAMARC_QA_SINGLE_DENOVOS always reject.
    if (test < 0.0)
    {
        if (test < log(randomSource->Float())) // rejection
        {
#endif
            tree->CopyPartialBody(oldtree);
            tree->CopyStick(oldtree);
            return false;
#ifndef LAMARC_QA_SINGLE_DENOVOS        // Make LAMARC_QA_SINGLE_DENOVOS always reject.
        }
    }
    // acceptance
    oldtree->CopyPartialBody(tree);
    oldtree->CopyStick(tree);
    chstate.TreeChanged();
    return true;
#endif
} // AcceptAndSynchronize

//------------------------------------------------------------------------------------

double ResimArranger::Hastings(ChainState & chstate)
{
    Tree * tree = chstate.GetTree();
    Tree * oldtree = chstate.GetOldTree();

    return log(static_cast<double>(oldtree->GetTimeList().GetNCuttable() - 1)
               / (tree->GetTimeList().GetNCuttable() - 1));
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* DropArranger::Clone() const
{
    Arranger* arr = new DropArranger(*this);
    return arr;
} // DropArranger::Clone

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* DenovoArranger::Clone() const
{
    Arranger* arr = new DenovoArranger(*this);
    return arr;
} // DenovoArranger::Clone

//------------------------------------------------------------------------------------

void DenovoArranger::Rearrange(ChainState & chstate)
// routine formerly known as ResimArranger::DenovoTree
{
    m_tree = chstate.GetTree();
    vector<Branch_ptr> tips = m_tree->ActivateTips(chstate.GetOldTree());

    // set up the active-lineage list
    m_activelist.Clear();
    m_inactivelist.Clear();
    unsigned long i;

    for (i = 0; i < tips.size(); ++i)
    {
        m_activelist.Append(tips[i]);
    }

    DropAll(0.0);
} // DenovoArranger::Rearrange

//------------------------------------------------------------------------------------

bool DenovoArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    // Denovo trees are always accepted unless the 'badtree' flag forbids
    if (badtree)
    {
        chstate.OverwriteTree();
        return false;
    }
    else
    {
        chstate.OverwriteOldTree();
        chstate.TreeChanged();
        return true;
    }
} // DenovoArranger::AcceptAndSynchronize

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

void BaseHapArranger::SetParameters(ChainState & chainstate)
{
    // Deliberately blank.
}

//------------------------------------------------------------------------------------

void BaseHapArranger::ScoreRearrangement(ChainState & chstate)
{
    chstate.GetTree()->CalculateDataLikes();
} // ScoreRearrangement

//------------------------------------------------------------------------------------
// We copy the full tree, as we may have changed the tips as well
// as the state of internal nodes (i.e. by changing DLCells).

bool BaseHapArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    Tree * tree = chstate.GetTree();
    Tree * oldtree = chstate.GetOldTree();

    if (badtree)     // reject this tree immediately
    {
        tree->CopyTips(oldtree);
        tree->CopyBody(oldtree);
        return false;
    }

    double test = (tree->GetDLValue() - oldtree->GetDLValue()) / temperature;

    if (test < 0.0)
    {
        if (test < log(randomSource->Float()))
        {
            tree->CopyTips(oldtree);
            tree->CopyBody(oldtree);
            return false;
        }
    }

    oldtree->CopyTips(tree);
    oldtree->CopyBody(tree);
    chstate.TreeChanged();
    return true;
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* HapArranger::Clone() const
{
    Arranger* arr = new HapArranger(*this);
    return arr;
} // HapArranger::Clone

//------------------------------------------------------------------------------------

void HapArranger::Rearrange(ChainState & chstate)
{
    m_tree = chstate.GetTree();
    m_tree->SwapSiteDLs();
    assert(m_tree->IsValidTree());
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* ProbHapArranger::Clone() const
{
    Arranger* arr = new ProbHapArranger(*this);
    return arr;
} // ProbHapArranger::Clone

//------------------------------------------------------------------------------------

void ProbHapArranger::Rearrange(ChainState & chstate)
{
    m_tree = chstate.GetTree();
    m_tree->PickNewSiteDLs();
    assert(m_tree->IsValidTree());
}

//------------------------------------------------------------------------------------

void ProbHapArranger::ScoreRearrangement(ChainState & chstate)
{
    BaseHapArranger::ScoreRearrangement(chstate);
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

TreeSizeArranger::TreeSizeArranger(const ForceSummary & fs, double timing)
    : ResimArranger(fs, timing)
{
    //We only want Active events, so we re-set m_eventvec.
    vector<Event*>::iterator evit;

    // Epoch times are handled in a different way, in Resimulate()
    for (evit = m_eventvec.begin(); evit != m_eventvec.end(); )
    {
        if ((*evit)->IsInactive() || (*evit)->IsEpoch())
        {
            delete *evit;
            evit = m_eventvec.erase(evit);
        }
        else
        {
            evit++;
        }
    }
}

//------------------------------------------------------------------------------------

Arranger* TreeSizeArranger::Clone() const
{
    Arranger* arr = new TreeSizeArranger(*this);
    return arr;
} // TreeSizeArranger::Clone

//------------------------------------------------------------------------------------

double TreeSizeArranger::Activate(Tree * oldtree)
{
    Branch_ptr branch(m_tree->ChoosePreferentiallyTowardsRoot(oldtree));
    return branch->m_eventTime;
} // Activate

//------------------------------------------------------------------------------------

void TreeSizeArranger::Resimulate(double eventT, ChainState & chainstate)
{
    // The following loop draws new times for each event in the tree via resimulating a new event,
    // discarding everything but its time, and storing the new time.  We will push all the new times
    // into the tree in one fell swoop.  This minimizes the amount of work done while the tree is in
    // a time-inconsistent state.

    TimeList & newtreelist = m_tree->GetTimeList();

    // Find the start of the area to be modified.
    Branchiter startpoint;

    for(startpoint = newtreelist.FirstBody();
        startpoint != newtreelist.EndBranch(); startpoint = newtreelist.NextBody(startpoint))
    {
        if ((*startpoint)->m_eventTime >= eventT) break;
    }
    assert(startpoint != newtreelist.EndBranch());
    assert((*startpoint)->m_eventTime == eventT);

    // Set eventT so it defines the start of the interval in which the branch it belongs to resides.
    Branchiter intervalstart(startpoint);
    while ((*intervalstart)->m_eventTime == eventT)
        intervalstart = newtreelist.PrevBodyOrTip(intervalstart);

    eventT = (*intervalstart)->m_eventTime;
    assert((*startpoint)->m_eventTime > eventT);

    m_activelist.Clear();
    m_activelist.Append(m_tree->FindBranchesImmediatelyTipwardOf(startpoint));

    DoubleVec1d epochtimes(chainstate.GetParameters().GetEpochTimes());

    Branchiter newbranch;
    DoubleVec1d newtimes;
    for(newbranch = startpoint; newbranch != newtreelist.EndBranch(); newbranch = newtreelist.NextBody(newbranch))
    {
        double nextT;

        m_tree->SetCurTargetLinkweightFrom(m_activelist);

        if ((*newbranch)->Event() == btypeEpoch)
        {
            // Epoch branches retain their old times; we will throw if that leads to inconsistency.
            nextT = (*newbranch)->GetTime();
        }
        else
        {
            Event* eventptr;
            nextT = EventTime(eventptr, eventT, DBL_MAX);
            assert(eventptr != NULL);
            // did we illegally pass an epoch boundary
            if (NodeChangesEpoch(*newbranch, nextT, epochtimes))
            {
                m_tree->ClearCurTargetLinkweight();
                string wh("crossed the epoch boundary at time ");
                wh += ToString(eventT);
                epoch_error e("wh");
                throw e;
            }
        }
        newtimes.push_back(nextT);

        // Now to fix up the activelist--remove all the children (if any).
        for(long i = 0; i < NELEM; ++i)
        {
            Branch_ptr br((*newbranch)->Child(i));
            if (br) m_activelist.Remove(br);
        }
        // Then add the newbranch.
        m_activelist.Append(*newbranch);

        if ((*newbranch)->Event() == btypeRec)
        {
            newtimes.push_back(nextT);
            newbranch = newtreelist.NextBody(newbranch);
            assert((*newbranch)->Event() == btypeRec);
            m_activelist.Append(*newbranch);
        }

        eventT = nextT;
    }

    assert(m_tree->IsValidTree());

#ifndef NDEBUG
    vector<double> sorttimes(newtimes);
    stable_sort(sorttimes.begin(), sorttimes.end());
    assert(newtimes == sorttimes); // times should have been in order!
#endif

    m_tree->SetNewTimesFrom(startpoint, newtimes);
    assert(m_tree->IsValidTree());

    // clean up Tree scratchpads
    m_tree->ClearCurTargetLinkweight();

} // Resimulate

//------------------------------------------------------------------------------------

void TreeSizeArranger::CleanupAfterResimulate()
{
    m_tree->TrimStickToRoot();
} // CleanupAfterResimulate

//------------------------------------------------------------------------------------

bool TreeSizeArranger::NodeChangesEpoch(Branch_ptr pBranch, double newtime, const DoubleVec1d & epochtimes) const
{
    double tipwardtime(min(pBranch->m_eventTime, newtime)), rootwardtime(max(pBranch->m_eventTime, newtime));

    DoubleVec1d::size_type epoch;
    for(epoch = 0; epoch < epochtimes.size(); ++epoch)
    {
        // Does an epoch boundary lie between our two times?
        if (tipwardtime < epochtimes[epoch] && rootwardtime > epochtimes[epoch])
            return true;

    }

    return false;

} // NodeStaysInSameEpoch

//------------------------------------------------------------------------------------

double TreeSizeArranger::Hastings(ChainState & chstate)
{
    //The tree size arranger does not include a hastings term for the number of cuttable branches because
    // both should be identical.  Also, since we are choosing a random branch instead of a random time,
    // there need be no compensation for that, either.  The return value is ln(Hastings ratio), hence the 0.
    return 0.0;
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

EpochSizeArranger::EpochSizeArranger(const ForceSummary & fs, double timing)
    : ResimArranger(fs, timing),
      m_tree_changed(false)
{
    // If there is no DivForce, do nothing--this Arranger is unused.
    if (!fs.CheckForce(force_DIVERGENCE)) return;

    // Cache priors on epoch times for later use.
    m_epochs = fs.GetEpochs();
    ForceVec::const_iterator myforce = registry.GetForceSummary().GetForceByTag(force_DIVERGENCE);
    vector<Parameter> myparams = (*myforce)->GetParameters();
    unsigned long i;
    m_priors.clear();

    for (i = 0; i < myparams.size(); ++i)
    {
        m_priors.push_back(myparams[i].GetPrior());
    }
}

//------------------------------------------------------------------------------------

EpochSizeArranger::EpochSizeArranger(const EpochSizeArranger& src)
    : ResimArranger(src),
      m_tree_changed(src.m_tree_changed),
      m_epochs(src.m_epochs),
      m_priors(src.m_priors)
{
    // Deliberately blank.
} // EpochSizeArranger copy constructor

//------------------------------------------------------------------------------------

Arranger* EpochSizeArranger::Clone() const
{
    Arranger* arr = new EpochSizeArranger(*this);
    return arr;
}

//------------------------------------------------------------------------------------

void EpochSizeArranger::Rearrange(ChainState & chstate)
{
    // Rearrange may throw a "rejecttree_error" to signal a newtree that should be discarded.

    m_tree = chstate.GetTree();
    m_tree_changed = false;
    assert(m_tree->ConsistentWithParameters(chstate.GetParameters()));

    // Time 0 is not present in the result of GetEpochTimes, so we put it in!
    DoubleVec1d epochtimes(chstate.GetParameters().GetEpochTimes());
    epochtimes.insert(epochtimes.begin(), 0.0);

    // Pick among the n internal boundary times.  The first epoch has no
    // pickable boundary time, but the last one DOES; there is no trailing sentinal.
    DoubleVec1d::size_type startepoch(registry.GetRandom(). Long(epochtimes.size() - 1) + 1);

    bool islastepoch = (startepoch == epochtimes.size() - 1);
    double tipwardtime(epochtimes[startepoch - 1]);
    double oldtime(epochtimes[startepoch]);
    assert(oldtime != 0.0);  // zero is a fake epoch
    double rootwardtime;
    if (!islastepoch) rootwardtime = epochtimes[startepoch + 1];
    else rootwardtime = FLAGDOUBLE;     // We shouldn't be using this if we're moving the last epoch.

    // Obtain the prior for the boundary to be moved (note that the priors start with epoch 1.
    const Prior& prior = m_priors[startepoch - 1];
    // Draw from it.
    pair<double, double> draw = prior.RandomDraw();
    double newtime = draw.first;

    // If setting the epoch to the drawn time causes a conflict, throw.
    if (newtime <= tipwardtime || (rootwardtime != FLAGDOUBLE && newtime >= rootwardtime))
    {
        epoch_error e("Epoch boundary conflict in EpochSizeArranger");
        throw e;
    }

    epochtimes[startepoch] = newtime;

    // If the affected area is past the bottom of the tree, don't change the tree, just reset the time and continue.
    if (tipwardtime >= m_tree->RootTime())
    {
        epochtimes.erase(epochtimes.begin());
        chstate.GetParameters().SetEpochTimes(epochtimes);
        return;
    }

    // Else change the tree.
    m_tree_changed = true;

    // Set m_firstInvalid correctly.
    m_tree->ChooseFirstBranchInEpoch(tipwardtime, chstate.GetOldTree());

    double tipwardchange = (newtime - tipwardtime) / (oldtime - tipwardtime);
    double rootwardchange;
    if (islastepoch) rootwardchange = 1.0;
    else rootwardchange = (rootwardtime - newtime) / (rootwardtime - oldtime);

    bool movingtipward(newtime < oldtime);

    // First generate all the new branchtimes, storing for later use.
    DoubleVec1d newtimes;

    vector<Branch_ptr> tipwbranches(m_tree->FindBranchesStartingOnOpenInterval(tipwardtime, oldtime));
    vector<Branch_ptr>::iterator br;
    for(br = tipwbranches.begin(); br != tipwbranches.end(); ++br)
    {
        assert((*br)->Event() != btypeEpoch);
        double newlength((*br)->m_eventTime - tipwardtime);
        newlength *= tipwardchange;
        newtimes.push_back(tipwardtime + newlength);
    }
    vector<Branch_ptr> epochbranches(m_tree->FindEpochBranchesAt(oldtime));
    for(br = epochbranches.begin(); br != epochbranches.end(); ++br)
    {
        assert((*br)->Event() == btypeEpoch);
        newtimes.push_back(newtime);
    }

    vector<Branch_ptr> rootwbranches;
    if (islastepoch)
        rootwbranches = m_tree->FindBranchesStartingRootwardOf(oldtime);
    else
        rootwbranches = m_tree->FindBranchesStartingOnOpenInterval(oldtime, rootwardtime);

    for(br = rootwbranches.begin(); br != rootwbranches.end(); ++br)
    {
        assert((*br)->Event() != btypeEpoch);
        if (islastepoch) {
           newtimes.push_back((*br)->m_eventTime + (newtime - oldtime));
        } else {
           double newlength(rootwardtime - (*br)->m_eventTime);
           newlength *= rootwardchange;
           newtimes.push_back(rootwardtime - newlength);
        }
    }

    // Now put the new times into the tree.
    if (!newtimes.empty())
    {
        Branchiter startpoint;

        if (!tipwbranches.empty())
        {
            startpoint = m_tree->GetTimeList().FindIter(*(tipwbranches.begin()));
        }
        else if (!epochbranches.empty())
        {
            startpoint = m_tree->GetTimeList().FindIter(*(epochbranches.begin()));
        }
        else
            startpoint = m_tree->GetTimeList().FindIter(*(rootwbranches.begin()));

        m_tree->SetNewTimesFrom(startpoint, newtimes);
        m_tree->ClearCurTargetLinkweight();
    }

    assert(m_tree->IsValidTree());

    // Set the new epoch times time 0 is not wanted, so we take it out!
    epochtimes.erase(epochtimes.begin());
    chstate.GetParameters().SetEpochTimes(epochtimes);
    assert(m_tree->ConsistentWithParameters(chstate.GetParameters()));
} // Rearrange

//------------------------------------------------------------------------------------

void EpochSizeArranger::ScoreRearrangement(ChainState & chstate)
{
    // Calculate data likelihood of new tree (stores it in the Tree object).
    if (m_tree_changed) chstate.GetTree()->CalculateDataLikes();

} // ScoreRearrangement

//------------------------------------------------------------------------------------

double EpochSizeArranger::Hastings(ChainState & chstate)
{
    Tree *tree = chstate.GetTree();
    DoubleVec1d newepochs(chstate.GetParameters().GetEpochTimes());
    DoubleVec1d oldepochs(chstate.GetOldParameters().GetEpochTimes());

    assert(newepochs.size() == oldepochs.size());
    DoubleVec1d::size_type index;
    for(index = 0; index < newepochs.size(); ++index)
    {
       if (newepochs[index] != oldepochs[index]) break;
    }
    assert(index < newepochs.size()); // ran off the end!

    if (index == newepochs.size()) return 0.0;

//  The hastings ratio is of the form Ratio#1 x Ratio#2, where Ratio#2
//  is just the digit one if the new epoch boundary is the most rootward
//  epoch boundary in the tree.
//
//  Ratio#1, used in all cases, starts as the quantity:
//     (newtau - tautipward) / (oldtau - tautipward),
//     where tautipward is either the time of the epoch boundary tipwards
//     of the changed epoch boundary, or zero if such a boundary does
//     not exist.  This ratio is then raised to the power of the number
//     of nodes in the tree between tauabove and newtau.
//  Ratio#2, used conditionally (see above), starts as the quantity:
//     (taurootward - newtau) / (taurootward - oldtau)
//     which is then raised to the power of the number of nodes in the
//     tree between newtau and taubelow.

    double tauratio(0.0),tiptau(0.0);
    long nodecount(0L);
    if (index != 0) tiptau = newepochs[index-1];
    tauratio = (newepochs[index] - tiptau) / (oldepochs[index] - tiptau);
    nodecount = tree->CountNodesBetween(tiptau,newepochs[index]);
    double loghratio(nodecount*log(tauratio));

    if (index != newepochs.size()-1)
    {
       assert(newepochs[index+1] == oldepochs[index+1]);
       tauratio = (newepochs[index+1] - newepochs[index]) /
                  (oldepochs[index+1] - oldepochs[index]);
       nodecount = tree->CountNodesBetween(newepochs[index],
                                            newepochs[index+1]);
       loghratio += nodecount*log(tauratio);
    }

    return loghratio;

} // Hastings
    
//------------------------------------------------------------------------------------

bool EpochSizeArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    // NB:  We assume that the tips of the tree were not changed, and thus we only accept/reject the body of the tree.

    if (badtree)    // reject this tree immediately
    {
        chstate.OverwriteTree();
        chstate.OverwriteParameters();
        return false;
    }

    // Sometimes an Epoch size change is past the bottom of the current tree and does nothing.  In this case,
    // we accept but without changing the tree or anything to do with it.  NB:  if this check is taken out
    // the program will DIE by trying to do data likelihood on an unchanged tree!
    if (!m_tree_changed)
    {
        chstate.OverwriteOldParameters();
        chstate.ParametersChanged();
        return true;
    }

    Tree * tree = chstate.GetTree();
    Tree * oldtree = chstate.GetOldTree();

    ForceParameters & newparameters = chstate.GetParameters();
    ForceParameters & oldparameters = chstate.GetOldParameters();
    SinglePostLike & postlike = registry.GetSinglePostLike();

    // Summarize tree; we end up owning this summary.
    // WARNING:  leaks memory if intervening code throws.
    TreeSummary * newtrsum = tree->SummarizeTree();
    TreeSummary * oldtrsum = oldtree->SummarizeTree();

    // Pass tree summary with old and new parameters to postlike routines this returns ln likelihood!
    double oldprior = postlike.Calc_lnProbGP(oldparameters.GetRegionalParameters(),
                                             oldparameters.GetRegionalLogParameters(),
                                             oldtrsum);
    double newprior = postlike.Calc_lnProbGP(newparameters.GetRegionalParameters(),
                                             newparameters.GetRegionalLogParameters(),
                                             newtrsum);
    //Postlike operates in Regional parameter space (unlike the priors, in Rearrange),
    // so we have to send them that version.

    // Delete the summary.
    delete newtrsum;
    delete oldtrsum;

#ifndef STATIONARIES
    double newlike(tree->GetDLValue() + newprior);
    double oldlike(oldtree->GetDLValue() + oldprior);
#else // STATIONARIES
    double newlike(newprior);
    double oldlike(oldprior);
#endif

    double test = (newlike - oldlike) / temperature;
    test += Hastings(chstate);

#if 0 // Always reject, JREMOVE/JRESTORE
    tree->CopyPartialBody(oldtree);
    tree->CopyStick(oldtree);
    chstate.OverwriteParameters();
    return false;
#endif

#if 0 // Always accept, JREMOVE/JRESTORE

#else
    if (test < 0.0)
    {
        if (test < log(randomSource->Float())) // rejection
        {
            tree->CopyPartialBody(oldtree);
            tree->CopyStick(oldtree);
            chstate.OverwriteParameters();
            return false;
        }
    }
#endif

    oldtree->CopyPartialBody(tree);     // acceptance
    oldtree->CopyStick(tree);
    chstate.OverwriteOldParameters();
    chstate.TreeChanged();
    chstate.ParametersChanged();
    return true;
} // AcceptAndSynchronize

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

BayesArranger::BayesArranger(const BayesArranger& src)
    : Arranger(src),
      m_oldlogs(src.m_oldlogs),
      m_newlogs(src.m_newlogs)
{
    // Deliberately blank.
} // copy ctor

//------------------------------------------------------------------------------------

Arranger* BayesArranger::Clone() const
{
    return new BayesArranger(*this);
} // Clone

//------------------------------------------------------------------------------------

void BayesArranger::SetParameters(ChainState & chainstate)
{
    // We make sure that oldlogs are ready for use, which means they start out equal to newlogs.
    m_newlogs = chainstate.GetParameters().GetRegionalLogParameters();
    m_oldlogs = m_newlogs;
}  // BayesArranger::SetParameters

//------------------------------------------------------------------------------------

void BayesArranger::Rearrange(ChainState & chstate)
{
    DoubleVec1d newparameters = chstate.GetParameters().GetGlobalParameters();
    //We must operate in global parameter space, since that's what space the prior operates in.

    // Choose a parameter.
    const ParamVector pv(true);
    long chosen = pv.ChooseSampleParameterIndex(randomSource);

    // Draw from appropriate prior.
    pair<double, double> newp = pv[chosen].DrawFromPrior();

    bool islog = false;
    registry.GetForceSummary().SetParamWithConstraints(chosen, newp.first, newparameters, islog);

    chstate.GetParameters().SetGlobalParameters(newparameters);
    chstate.UpdateNewStickParams(); // make sure stick reflects this new parameter
    double newlogregparam = chstate.GetParameters().GetRegionalLogParameter(chosen);

    // This may not be a log, if it's growth, but we let the ForceParameter worry about that.
    // We do need to make sure it's the regional parameter, not the global parameter.
    islog = true;
    registry.GetForceSummary().SetParamWithConstraints(chosen, newlogregparam, m_newlogs, islog);

}  // BayesArranger::Rearrange

//------------------------------------------------------------------------------------

void BayesArranger::ScoreRearrangement(ChainState & chstate)
{
    // It would be logical to compute the prior here, but currently
    // we do that in AcceptAndSynchronize.  OPTIMIZATION possible here.
} // ScoreRearrangement

//------------------------------------------------------------------------------------

bool BayesArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
#ifdef STATIONARIES

    chstate.OverwriteOldParameters();
    chstate.UpdateOldStickParams();
    m_oldlogs = m_newlogs;
    chstate.ParametersChanged();
    return true;

#else // STATIONARIES

    // MDEBUG OPTIMIZATION we reset the stick much more often than we need to, as it
    // is only invalidated by changes to Theta, but we reset it on any change.

    assert(!badtree);  // Bayesian arrangers never make bad trees, since they do not make trees at all!

    ForceParameters & newparameters = chstate.GetParameters();
    ForceParameters & oldparameters = chstate.GetOldParameters();
    SinglePostLike & postlike = registry.GetSinglePostLike();

    // Summarize tree; we end up owning this summary.
    // WARNING:  leaks memory if intervening code throws
    TreeSummary * trsum = chstate.GetTree()->SummarizeTree();

    // Pass tree summary with old and new parameters to postlike routines this returns ln likelihood!
    double oldprior = postlike.Calc_lnProbGP(oldparameters.GetRegionalParameters(), m_oldlogs, trsum);
    double newprior = postlike.Calc_lnProbGP(newparameters.GetRegionalParameters(), m_newlogs, trsum);

    // Postlike operates in Regional parameter space (unlike the priors, in Rearrange),
    // so we have to send them that version.

    // Delete the summary.
    delete trsum;

    // choose a winner
    double test = (newprior - oldprior) / temperature;

    if (test < log(randomSource->Float()))                                              // reject
    {
        chstate.OverwriteParameters();
        chstate.UpdateNewStickParams();
        m_newlogs = m_oldlogs;
        return false;
    }

    chstate.OverwriteOldParameters();                                                   // else accept
    chstate.UpdateOldStickParams();
    m_oldlogs = m_newlogs;
    chstate.ParametersChanged();

    return true;

#endif // STATIONARIES
} // BayesArranger::AcceptAndSynchronize

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

void LocusArranger::SetParameters(ChainState & chainstate)
{
    // Do nothing--the parameters don't change in a moving locus rearrangement.
}

//------------------------------------------------------------------------------------
// Our Locus arranger is a Gibbs arranger--it always accepts.

void LocusArranger::Rearrange(ChainState & chstate)
{
    RecTree * rtree = dynamic_cast<RecTree *>(chstate.GetTree());
    for (unsigned long mloc = 0; mloc < rtree->GetNumMovingLoci(); ++mloc)
    {
        //We might have a mix of jumping and floating loci--only do this for the jumping ones.
        if (rtree->DoesThisLocusJump(mloc))
        {
            DoubleVec1d likelihoods = rtree->CalculateDataLikesForFloatingLocus(mloc);
            ScaleLargestToZero(likelihoods);
            DoubleVec1d likesums;
            double total = 0;
            for (unsigned long site = 0; site < likelihoods.size(); ++site)
            {
                total += exp(likelihoods[site]);
                likesums.push_back(total);
            }
            double choice = randomSource->Float() * total;
            for (unsigned long site = 0; site < likesums.size(); ++site)
            {
                //LS DEBUG:  inefficient.  But earlier attempts were wrong, so.
                rtree->SetMovingMapPosition(mloc, site);
                if (likesums[site] > choice)
                {
                    break;
                }
            }
        }
    }
}

//------------------------------------------------------------------------------------

void LocusArranger::ScoreRearrangement(ChainState & chstate)
{
    // Calculate data likelihood of new tree (stores it in the Tree object).
    RecTree * rtree = dynamic_cast<RecTree *>(chstate.GetTree());
    rtree->GetTimeList().SetAllUpdateDLs();
    for (unsigned long mloc = 0; mloc < rtree->GetNumMovingLoci(); ++mloc)
    {
        rtree->CalculateDataLikesForMovingLocus(mloc);
    }

    //We need to fill the DLCells with appropriate values at the new location for the next arranger.
    // If we choose later to only rearrange a single randomly-chosen moving loci instead of all of them,
    // we'll need to store the number of the chosen locus and only CalculateDataLikesFor... for
    // that locus.
}

//------------------------------------------------------------------------------------

bool LocusArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    assert(!badtree);  //Enh?
    chstate.MapChanged();
    return true;
    //Note:  We don't do any tree copying, since the tree itself doesn't own what we changed.
    // So it's a good thing we always accept!
}

//------------------------------------------------------------------------------------

Arranger* LocusArranger::Clone() const
{
    Arranger* arr = new LocusArranger(*this);
    return arr;
}

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* ZilchArranger::Clone() const
{
    Arranger* arr = new ZilchArranger(*this);
    return arr;

} // ZilchArranger::Clone

//------------------------------------------------------------------------------------

bool ZilchArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    assert(!badtree);                   //What?  We did nothing!
    //We assume the trees are still identical.
    return false;                       //No sense fooling the user into thinking we did something.

} // ZilchArranger::AcceptAndSynchronize

//------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------

Arranger* StairArranger::Clone() const
{
    Arranger* arr = new StairArranger(*this);
    return arr;
} // Clone

//------------------------------------------------------------------------------------

//  double StickMean(double freqA, double s, double toA, double toa) const;
//  double StickVar(double freqA, double tipfreqA, double thetaA) const;

double StairArranger::LnPBrownStickParams(const DoubleVec1d & param, const TreeSummary * partialtreedata) const
{
    const StickSelectPL& plf(registry.GetSinglePostLike().GetStickSelectPL());
    double answ(0.0);

    (void)plf;                          // Silence compiler warning about unused variable.

    return answ;

} // LnPBrownStickParams

//------------------------------------------------------------------------------------

void StairArranger::SetParameters(ChainState & chainstate)
{
    // Get the parameters we need, whatever those are.
} // SetParameters

//------------------------------------------------------------------------------------

void StairArranger::Rearrange(ChainState & chstate)
{
    ForceParameters fp(chstate.GetParameters());
    m_tree = chstate.GetTree();

    m_tree->GetTimeManager()->MakeStickTilTime(fp, m_tree->RootTime());

} // Rearrange

//------------------------------------------------------------------------------------

void StairArranger::ScoreRearrangement(ChainState & chstate)
{
    // In ResimulatingArrangers this routine scores P(D|G).  The analogous computation in StairArranger,
    // like BayesArranger, is done in AcceptAndSynchronize, so that even when stationaries are being run,
    // it will still happen.  Therefore this routine is empty.
} // ScoreRearrangement

//------------------------------------------------------------------------------------
// Much of this code is cut and paste from BayesArranger and should probably be combined,
// but not all of it, so this function can't simply be shared or inherited.

bool StairArranger::AcceptAndSynchronize(ChainState & chstate, double temperature, bool badtree)
{
    assert(!badtree);                   // We don't make trees at all, how could it be bad?!

    // Since we Gibbs sample, it's always accept...

#ifndef STATIONARIES
    // Summarize trees; we end up owning these summaries.
    // WARNING:  leaks memory if intervening code throws.

    // NB:  while it looks like we're comparing trees here, the trees differ only in their
    // stairs.  We pass a whole TreeSummary only because there is no separate StairSummary.

    TreeSummary * newtrsum = chstate.GetTree()->SummarizeTree();
    TreeSummary * oldtrsum = chstate.GetOldTree()->SummarizeTree();

    SinglePostLike & postlike = registry.GetSinglePostLike();
    DoubleVec1d param = chstate.GetParameters().GetRegionalParameters();
    DoubleVec1d logparam = chstate.GetParameters().GetRegionalLogParameters();
    double oldprior = postlike.Calc_lnProbGS(param, logparam, oldtrsum);
    double newprior = postlike.Calc_lnProbGS(param, logparam, newtrsum);
    //  double oldprior = postlike.Calc_lnProbGP(param, logparam, oldtrsum);
    //  double newprior = postlike.Calc_lnProbGP(param, logparam, newtrsum);

    // Delete the summaries.
    delete newtrsum;
    delete oldtrsum;

    // Choose the winner.
    double test = (newprior - oldprior) / temperature;

    if (test < log(randomSource->Float())) // reject
    {
        chstate.GetTree()->CopyStick(chstate.GetOldTree());
        return false;
    }
#endif // STATIONARIES

    chstate.GetOldTree()->CopyStick(chstate.GetTree()); // else accept
    chstate.StickChanged();
    return true;

} // ScoreRearrangement

//____________________________________________________________________________________
