// $Id: parsetreetodata.cpp,v 1.58 2018/01/03 21:33:06 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Peter Beerli, Mary Kuhner, Jon Yamato and Joseph Felsenstein */


#include <cassert>
#include <boost/algorithm/string.hpp>

#include "argtree.h"
#include "datapack.h"
#include "datatype.h"                   // to set the datatype of a region and to proofread data
#include "errhandling.h"
#include "newick.h"                     // for UserTree and subclasses thereof
#include "parsetreetodata.h"
#include "random.h"
#include "region.h"
#include "registry.h"
#include "stringx.h"
#include "timex.h"
#include "tree.h"
#include "types.h"                      // for access to DataType's auto_ptr type
#include "ui_strings.h"
#include "xml.h"
#include "xml_strings.h"

using std::string;

//------------------------------------------------------------------------------------

ParseTreeToData::ParseTreeToData(XmlParser& parser, DataPack& dp)
    :
    ParseTreeWalker(parser),
    datapack(dp),
    m_pCurrRegion(NULL),
    m_currpopulation(FLAGLONG),
    m_currindno(FLAGLONG),
    m_randomNameSource(new Random()),
    m_migrationalForce(force_NONE)
{
    m_randomNameSource->Seed(GetTime());
}

//------------------------------------------------------------------------------------

ParseTreeToData::~ParseTreeToData()
{
}

//------------------------------------------------------------------------------------

string
ParseTreeToData::XmlRandomLongAsString()
{
    return ToString(m_randomNameSource->Long(XML_RANDOM_NAME_LENGTH));
}

//------------------------------------------------------------------------------------

void
ParseTreeToData::ProcessFileData()
{
    // MCHECK
    TiXmlElement* docElement = m_parser.GetRootElement();
    const char * tagValue = docElement->Value();
    string tagString(tagValue);
    bool matches = CaselessStrCmp(xmlstr::XML_TAG_LAMARC,tagValue);
    if(!matches)
    {
        string msg = m_parser.GetFileName() + ": " + xmlstr::XML_ERR_NOT_LAMARC;
        m_parser.ThrowDataError(msg);
    }

    // check to see if Divergence is in play
    DiagnoseTagForMigrationalForce(m_parser.GetRootElement());

    // parse the actual data
    TiXmlElement* format = singleOptionalChild(m_parser.GetRootElement(),xmlstr::XML_TAG_FORMAT);
    if (format != NULL)
    {
        DoFormat(format);
    }
    DoData(singleRequiredChild(m_parser.GetRootElement(),xmlstr::XML_TAG_DATA));

    // now do some post-analysis of data

    // at the moment this checks if there are multiple pops, and if
    // there aren't it removes population info from the stored tips
    datapack.RemoveUneededPartitions();

    // if divergence is in effect, pre-parse Divergence force and modify DataPack
    // accordingly (increasing number of partitions and xpartitions)
    // parse down to <divergence> block if it exists
    TiXmlElement * forcesElement =
        singleOptionalChild(docElement,xmlstr::XML_TAG_FORCES);
    if(forcesElement != NULL)
    {
        TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_DIVERGENCE);
        if (forceElement != NULL)
        {
            TiXmlElement * epochsElement = singleRequiredChild(forceElement,xmlstr::XML_TAG_POPTREE);
            TiXmlNode * child = NULL;
            while((child = epochsElement->IterateChildren(xmlstr::XML_TAG_EPOCH_BOUNDARY, child)))
            {
                TiXmlElement * boundaryElement = child->ToElement();
                TiXmlElement * ancestorElement =
                    singleOptionalChild(boundaryElement,xmlstr::XML_TAG_ANCESTOR);
                string epochname = getNodeText(ancestorElement);
                datapack.AddPartition(force_DIVMIG,epochname);
            }
        }
    }
}

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoFormat(TiXmlElement * formatElement)
{
    TiXmlElement* convertElement = singleOptionalChild(formatElement, xmlstr::XML_TAG_CONVERT_OUTPUT);
    if (convertElement != NULL)
    {
        string convert = getNodeText(convertElement);
        registry.SetConvertOutputToEliminateZeroes(ProduceBoolOrBarf(convert));
    }
}

void
ParseTreeToData::DoData(TiXmlElement * dataElement)
{

    TiXmlNode * child = NULL;
    long regionNumber = 0;

    while((child = dataElement->IterateChildren(xmlstr::XML_TAG_REGION,child)))
    {
        TiXmlElement * regionElement = child->ToElement();
        DoRegion(regionElement,regionNumber);
        regionNumber++;
    }

} // DoData

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoRegion(TiXmlElement * regionElement, long regionId)
{

    // set up a region
    string regname = getNodeAttributeValue(regionElement,xmlstr::XML_ATTRTYPE_NAME);
    if (datapack.IsDuplicateRegionName(regname))
    {
        string problem(xmlstr::XML_ERR_DUPLICATE_REGIONNAME_0);
        problem += regname + xmlstr::XML_ERR_DUPLICATE_REGIONNAME_1;
        m_parser.ThrowDataError(problem);
    }
    m_pCurrRegion = new Region(regname);
    datapack.SetRegion(m_pCurrRegion);
    m_currindno = 0;
    m_pCurrRegion->SetSnpPanel(false);

    DoPopulations(regionElement);

    DoSpacing(singleOptionalChild(regionElement,xmlstr::XML_TAG_SPACING));

    const Region *cnst_Reg =  m_pCurrRegion;// jrmhack
    const Individual& testind2 = cnst_Reg->GetIndividual(0); // jrmhack
    DoPhases(regionElement);

    const Individual& testind3 = cnst_Reg->GetIndividual(0); // jrmhack
    DoEffectivePopSize(
        singleOptionalChild(regionElement,xmlstr::XML_TAG_EFFECTIVE_POPSIZE)
        );

    TiXmlElement * treeElement
        = singleOptionalChild(regionElement,xmlstr::XML_TAG_TREE);
    if (treeElement != NULL)
    {
        string treetype = getNodeAttributeValue(treeElement,xmlstr::XML_ATTRTYPE_TYPE);

        if (treetype == xmlstr::XML_STRING_NEWICK)
        {
            m_pCurrRegion->SetUserTree(DoTree(treeElement));
        }
        else if (treetype == xmlstr::XML_STRING_ARG)
        {
            registry.SetARGfound(true);
        }
    }

    string errorString;
    if (!m_pCurrRegion->IsValidRegion(errorString))
    {
        m_parser.ThrowDataError(xmlstr::XML_ERR_INCONSISTENT_REGION +
                                m_pCurrRegion->GetRegionName()
                                + xmlstr::XML_STRING_COLON + errorString
                                , regionElement->Row());
    }

} // DoRegion

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoPopulations(TiXmlElement* regionElement)
{

    TiXmlNode * child = NULL;
    while((child = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,child)))
    {
        TiXmlElement * populationElement = child->ToElement();
        DoPopulation(populationElement);
    }
}

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoPopulation(TiXmlElement * populationElement)
{
    string popname = getNodeAttributeValue(populationElement,xmlstr::XML_ATTRTYPE_NAME);
    m_currpopulation = datapack.AddPartition(TagForMigrationalForce(),popname);

    TiXmlNode * child = NULL;
    while((child = populationElement->IterateChildren(xmlstr::XML_TAG_INDIVIDUAL,child)))
    {
        TiXmlElement * individualElement = child->ToElement();
        DoIndividual(individualElement);
    }

    while((child = populationElement->IterateChildren(xmlstr::XML_TAG_PANEL,child)))
    {
        TiXmlElement * panelElement = child->ToElement();
        DoPanel(panelElement, popname);
    }
}

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoIndividual(TiXmlElement * individualElement)
{
    Individual newind;
    m_currIndividual = newind; //barring a .clear() function or the like.
    m_currIndividual.SetId(m_currindno);

    string name = getNodeAttributeValue(individualElement,xmlstr::XML_ATTRTYPE_NAME);
    if (name.empty())
    {
        name = ToString(m_currindno);
    }
    m_currIndividual.SetName(name);

    DoSamples(individualElement);

    DoGenotypeResolutions(individualElement);
    m_pCurrRegion->AddIndividual(m_currIndividual);

    m_currindno++;
}

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoPanel(TiXmlElement * panelElement, string popname)
{
    // set up panel name
    string regionname = m_pCurrRegion->GetRegionName();
    string panelname = getNodeAttributeValue(panelElement,xmlstr::XML_ATTRTYPE_NAME);
    if (panelname.empty())
        panelname = "panel";
    string rootname = regionname + ToString('_');
    rootname += popname;
    rootname += ToString('_');
    rootname += panelname;

    // get panel size
    TiXmlNode * child = NULL;
    int panelsize = 0;
    while((child = panelElement->IterateChildren(xmlstr::XML_TAG_PANELSIZE,child)))
    {
        TiXmlElement * panelSizeElement = child->ToElement();
        string contents = getNodeText(panelSizeElement);
        panelsize = atoi(contents.c_str());
    }

    if (panelsize <= 0)
    {
        string problem = "Invalid panel count: ";
        problem += ToString(panelsize);
        problem += " in <Panel-size>";
        m_parser.ThrowDataError(problem);

    }

    // turn on panel flag
    m_pCurrRegion->SetSnpPanel(true);

    long nloc = m_pCurrRegion->GetNloci();

    Individual newind;

    long whichhap = 0; //???
    for (int i=0; i<panelsize; i++)
    {
        string panelname = rootname + "_p";
        panelname += ToString(i);
        m_currIndividual = newind; //barring a .clear() function or the like.
        m_currIndividual.SetId(m_currindno);
        m_currIndividual.SetName(panelname);

        for (long locus=0; locus<nloc; locus++)
        {
            // define tip
            Locus loc =  m_pCurrRegion->GetLocus(locus);
            long nsites = loc.GetNsites();
            string tval;
            tval.assign(nsites, '?');

            // copied from DoSamples
            m_tipdata.Clear();  // prepare m_tipdata for new data
            m_tipdata.label = panelname;
            m_tipdata.SetDataSource("panel");
            m_tipdata.AddPartition(
                std::make_pair(TagForMigrationalForce(),
                               datapack.GetPartitionName(
                                   TagForMigrationalForce(),m_currpopulation)
                    ));
            m_tipdata.m_popname = datapack.
                GetPartitionName(TagForMigrationalForce(),
                                 m_currpopulation);
            // NB--we redundantly store this because
            // the partitions of AddPartition are for branch use and don't
            // treat migration specially--this one is for xml output use
            // which currently still does treat migration specially from
            // other partitions (<status> in xml terms).
            m_tipdata.individual = m_currIndividual.GetId();
            m_tipdata.m_hap = whichhap;
            m_tipdata.m_locus = locus;

            // get block datatype
            string datatype = ToString(loc.GetDataTypePtr()->GetType());
            DataType_ptr dt(CreateDataType(datatype));
            string baddata;
            bool good_data = dt->Proofread(tval, m_tipdata.data, baddata);
            if (!good_data)
            {
                string problem = "Invalid genetic data in population "
                    + ToString(m_currpopulation)
                    + " sample "
                    + m_tipdata.label
                    + "\n";
                problem += "Offending allele: " + baddata;
                m_parser.ThrowDataError(problem);
            }
            m_pCurrRegion->SetTipData(locus, m_tipdata);
            // ++whichhap; //???
        }
        m_pCurrRegion->AddIndividual(m_currIndividual);
        m_currindno++;
    }
} // DoPanels

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoSamples(TiXmlElement * individualElement)
{
    long whichhap = 0;
    TiXmlNode * child = NULL;
    vector<TiXmlElement*> samples = getAllOptionalDescendantElements(individualElement, xmlstr::XML_TAG_SAMPLE);
    m_pCurrRegion->AddPloidy(samples.size());
    while((child = individualElement->IterateChildren(xmlstr::XML_TAG_SAMPLE,child)))
    {
        TiXmlElement * sampleElement = child->ToElement();
        string name = getNodeAttributeValue(sampleElement,xmlstr::XML_ATTRTYPE_NAME);
        if (name.empty())
            name = m_currIndividual.GetName() + XmlRandomLongAsString();
        if (m_pCurrRegion->IsDuplicateTipName(name))
        {
            string problem(xmlstr::XML_ERR_DUPLICATE_SAMPLENAME_0);
            problem += name + xmlstr::XML_ERR_DUPLICATE_SAMPLENAME_1;
            problem += m_pCurrRegion->GetRegionName();
            m_parser.ThrowDataError(problem);
        }
        m_tipdata.Clear();  // prepare m_tipdata for new data
        m_tipdata.label = name;
        m_tipdata.AddPartition(
            std::make_pair(TagForMigrationalForce(),
                           datapack.GetPartitionName(TagForMigrationalForce(),m_currpopulation))
            );
        m_tipdata.m_popname = datapack.GetPartitionName(TagForMigrationalForce(),
                                                        m_currpopulation);  
        // NB--we redundantly store this because
        // the partitions of AddPartition are for branch use and don't
        // treat migration specially--this one is for xml output use
        // which currently still does treat migration specially from
        // other partitions (<status> in xml terms).
        m_tipdata.individual = m_currIndividual.GetId();
        m_tipdata.m_hap = whichhap;

        DoStatus(singleOptionalChild(sampleElement,xmlstr::XML_TAG_STATUS));
        DoDataBlocks(sampleElement);    ++whichhap;
    }
} // DoSamples

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoStatus(TiXmlElement * statusElement)
{
    if(statusElement != NULL)
    {
        TiXmlNode * child = NULL;
        while((child = statusElement->IterateChildren(xmlstr::XML_TAG_DISEASESTATUS,child)))
        {
            TiXmlElement * diseaseElement = child->ToElement();
            DoDisease(diseaseElement);
        }
    }
} // DoStatus

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoDisease(TiXmlElement * diseaseElement)
{
    // remove leading whitespace from dstatus
    string dstatus = getNodeText(diseaseElement);
    string whitespace(" ");
    long firstnonwhite = dstatus.find_first_not_of(whitespace);
    dstatus.assign(dstatus,firstnonwhite,dstatus.length()-firstnonwhite);

    datapack.AddPartition(force_DISEASE,dstatus);
    m_tipdata.AddPartition(std::make_pair(force_DISEASE,dstatus));

} // DoDisease

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoDataBlocks(TiXmlElement * sampleElement)
{
    long locus = 0;

    TiXmlNode * child = NULL;
    while((child = sampleElement->IterateChildren(xmlstr::XML_TAG_DATABLOCK,child)))
    {
        TiXmlElement * dataBlockElement = child->ToElement();

        // if this is a new locus, add it to the Region
        if (locus == m_pCurrRegion->GetNloci())
        {
            m_pCurrRegion->AddLocus();
        }
        assert(locus < m_pCurrRegion->GetNloci());

        // get block datatype
        string datatype = getNodeAttributeValue(dataBlockElement,xmlstr::XML_ATTRTYPE_TYPE);
        DataType_ptr dt(CreateDataType(datatype));
        string datasource = getNodeAttributeValue(dataBlockElement,xmlstr::XML_ATTRTYPE_SOURCE);

        m_pCurrRegion->SetDataType(locus, dt);

        // Read the data
        m_tipdata.data.clear();
        string contents = getNodeText(dataBlockElement);
        string baddata;
        bool good_data = dt->Proofread(contents, m_tipdata.data, baddata);
        if (!good_data)
        {
            string problem = "Invalid genetic data in population " + ToString(m_currpopulation)
                + " sample " + m_tipdata.label + "\n";
            problem += "Offending allele: " + baddata;
            m_parser.ThrowDataError(problem);
        }
        m_tipdata.SetDataSource(datasource);
        if((CaselessStrCmp(datatype, lamarcstrings::SNP)) &&
           (CaselessStrCmp(datasource, lamarcstrings::PANEL)))
            m_pCurrRegion->SetSnpPanel(true);

        long len = m_tipdata.data.size();

        try
        {
            m_pCurrRegion->SetNmarkers(locus, len);
        }
        catch (const std::exception& e)
        {
            long expectedNmarkers = m_pCurrRegion->GetNmarkers(locus);
            long localLineNumber = dataBlockElement->Row();
            string tagName = xmlstr::XML_TAG_DATABLOCK;
            string message = string ("Data element \"" + tagName + "\" at input line ")
                + ToString(localLineNumber) + string( " has ") + ToString(len);
            message += " data elements but previous has " + ToString(expectedNmarkers);
            m_parser.ThrowDataError (message);
        }
        // save info on which locus this tipdata belongs to
        m_tipdata.m_locus = locus;
        // put this info into the storehouse
        m_pCurrRegion->SetTipData(locus, m_tipdata);
        ++locus;
    }
} // DoDataBlocks

//------------------------------------------------------------------------------------

void ParseTreeToData::DoGenotypeResolutions(TiXmlElement * individualElement)
{
    TiXmlNode * child = NULL;
    while((child = individualElement->IterateChildren(xmlstr::XML_TAG_GENOTYPE_RESOLUTIONS,child)))
    {
        TiXmlElement * genotypeResElement = child->ToElement();
        TiXmlElement * traitElement = singleRequiredChild(genotypeResElement, xmlstr::XML_TAG_TRAIT_NAME);
        string tname = getNodeText(traitElement);

        // if this is a new locus, add it to the Region
        if (!(m_pCurrRegion->HasLocus(tname)))
        {
            m_pCurrRegion->AddLocus(true, tname); //true == movable (for later)
            //Note:  passing 'true' to the locus constructor sets up a locus
            // with a single marker and one site.
        }

        // Copy the vector of TipDatas from an older locus to the new one,
        //  and clear out the data from it.  This is because we don't have
        //  information about the samples here, only when we read in the sample
        //  data (which we already did).
        //
        // We need to do this every time instead of just the first time because
        //  this code gets run once per individual.  Ideally, we'd only run it the
        //  *last* time, but running it every time works.
        //
        // This is, uh, kind of a hack.  Like a lot of this code.  Ideally, *all*
        //  the TipData stuff would just be in the Individuals.
        m_pCurrRegion->CopyTipDataForLocus(tname);
        long locusnum = m_pCurrRegion->GetLocusIndex(tname);
        assert(locusnum < m_pCurrRegion->GetNloci());

        // The datatype must be K-allele (for now)
        // Also, this would be a good thing to move to the constructor
        // eventually.  Because as it is, we have to re-set this for
        // every individual.  But whatever.
        DataType_ptr dt(CreateDataType(lamarcstrings::KALLELE));
        m_pCurrRegion->SetDataType(locusnum, dt);

        // Read the haplotypes

        TiXmlNode * genchild = NULL;

        while ((genchild = genotypeResElement->IterateChildren(xmlstr::XML_TAG_HAPLOTYPES,genchild)))
        {
            TiXmlElement* haplotypesElement = genchild->ToElement();
            TiXmlElement* penetranceElement = singleOptionalChild(haplotypesElement, xmlstr::XML_TAG_PENETRANCE);
            TiXmlElement* allelesElement    = singleRequiredChild(haplotypesElement, xmlstr::XML_TAG_ALLELES);
            double penetrance = 1.0;
            if (penetranceElement != NULL)
            {
                penetrance = ProduceDoubleOrBarf(getNodeText(penetranceElement));
            }
            string baddata;
            StringVec1d alleles;
            dt->Proofread(getNodeText(allelesElement), alleles, baddata);
            //Kallele data always returns true;
            m_currIndividual.AddHaplotype(m_pCurrRegion->GetID(), tname, 0, alleles, penetrance);
        }
    }
}

//------------------------------------------------------------------------------------
// The following routines are used to establish the map
//------------------------------------------------------------------------------------

void
ParseTreeToData::DoSpacing(TiXmlElement * spacingElement)
{
    // NB: DoSpacing has to be called *after* DoPopulations
    // because otherwise defaults can't be meaningfully set here
    if (spacingElement == NULL)
    {
        long locus;
        for (locus = 0; locus < m_pCurrRegion->GetNloci(); ++locus)
        {
            m_pCurrRegion->SetPositions(locus);  // set to default
            m_pCurrRegion->SetNsites(locus, m_pCurrRegion->GetNmarkers(locus));
        }
        // all other spacing parameters are set by default in constructor
    }
    else
    {
        DoBlocks(spacingElement);
    }

} // DoSpacing

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoBlocks(TiXmlElement * spacingElement)
{
    long locus = 0;

    TiXmlNode * child = NULL;
    while((child = spacingElement->IterateChildren(xmlstr::XML_TAG_BLOCK,child)))
    {
        TiXmlElement * blockElement = child->ToElement();

        string locusName = getNodeAttributeValue(blockElement,xmlstr::XML_ATTRTYPE_NAME);
        if (!locusName.empty())
        {
            m_pCurrRegion->SetName(locus, locusName);
        }

        // Keep DoOffset before DoMapPosition, as it is the default for single locus map position
        DoOffset(singleOptionalChild(blockElement,xmlstr::XML_TAG_OFFSET),locus);
        DoMapPosition(singleOptionalChild(blockElement,xmlstr::XML_TAG_MAP_POSITION),locus);
        DoLength(singleOptionalChild(blockElement,xmlstr::XML_TAG_LENGTH),locus);
        DoLocations(singleOptionalChild(blockElement,xmlstr::XML_TAG_LOCATIONS),locus);

        ++locus;
    }
    assert(locus > 0);  // should be assured by schema logic
} // DoBlocks

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoMapPosition(TiXmlElement * mapPositionElement, long locus)
{

    long mapPosition = m_pCurrRegion->GetOffset(locus);     // default value if mapPositionElement is Null

    if (mapPositionElement != NULL) // no mapPosition; assume 1
    {
        try
        {
            mapPosition = ProduceLongOrBarf(getNodeText(mapPositionElement));
        }
        catch (const data_error& e)
        {
            m_parser.ThrowDataError(string(e.what()),mapPositionElement->Row());
        }
    }
    m_pCurrRegion->SetGlobalMapPosition(locus, mapPosition);

} // DoMapPosition

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoLength(TiXmlElement * lengthElement, long locus)
{
    // default value if lengthElement is Null
    long length = m_pCurrRegion->GetNmarkers(locus);

    if (lengthElement != NULL) // no mapPosition; assume 0
    {
        try
        {
            length = ProduceLongOrBarf(getNodeText(lengthElement));
        }
        catch (const data_error& e)
        {
            m_parser.ThrowDataError(string(e.what()),lengthElement->Row());
        }
    }
    m_pCurrRegion->SetNsites(locus, length);

} // DoLength

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoLocations(TiXmlElement * locationsElement, long locus)
{
    if(locationsElement == NULL)
    {
        m_pCurrRegion->SetPositions(locus);  // to default
    }
    else
    {
        string content = getNodeText(locationsElement);
        vector<long> locations;
        string token;
        long offset = m_pCurrRegion->GetOffset(locus);
        long length = m_pCurrRegion->GetLocusNsites(locus);
        long nmarkers = m_pCurrRegion->GetNmarkers(locus);

        try
        {
            StringVec1d values = getNodeTextSplitOnWhitespace(locationsElement);
            if (static_cast<size_t>(nmarkers) != values.size())
            {
                throw data_error("Incorrect number of locations listed (" + ToString(values.size()) + "); you should have " + ToString(nmarkers) + ".");
            }

            StringVec1d::iterator siter;
            for(siter = values.begin(); siter != values.end(); siter++)
            {
                // we subtract the offset here, making all positions
                // relative to zero
                long pos = ProduceLongOrBarf(*siter) - offset;
                if (pos < 0)
                {
                    throw data_error("One or more location values are lower than " + ToString(offset) + ", the offset for locus " + ToString(locus+1) + ".");
                }
                if (pos >= length)
                {
                    throw data_error("One or more location values are higher than " + ToString(length + offset - 1) + ", the offset plus the length of locus " + ToString(locus+1) + ".");
                }
                // we subtract the offset here, making all positions
                // relative to zero
                locations.push_back(pos);
            }
            m_pCurrRegion->SetPositions(locus, locations);
        }
        catch (const data_error& e)
        {
            m_parser.ThrowDataError(string(e.what()),locationsElement->Row());
        }
    }
} // DoLocations

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoOffset(TiXmlElement * offsetElement, long locus)
{
    long offset = 0L;     // default value if offsetElement is Null

    if (offsetElement != NULL) // no offset; assume 0
    {
        try
        {
            offset = ProduceLongOrBarf(getNodeText(offsetElement));
        }
        catch (const data_error& e)
        {
            m_parser.ThrowDataError(string(e.what()),offsetElement->Row());
        }
    }

    m_pCurrRegion->SetOffset(locus, offset);

} // DoOffset

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoPhases(TiXmlElement * regionElement)
// read information about sites of known/unknown phase
// attribute "unknown" means that this is a list of phase-unknown
// sites (to be stored as-is) whereas attribute "known" means that
// this is a list of phase-known sites (and we store the
// inverse, the phase-unknown sites).
{
    //Because populations have to be done before spacing, and because phases
    // have to be done after spacing, we have to loop over exactly the same
    // things that 'DoPopulations' did, before, to get to the individuals.
    TiXmlNode * pop = NULL;
    long indnum = 0;

    // panel members don't have phases so create an empty vector
    LongVec2d panelsites(m_pCurrRegion->GetNumFixedLoci());
    int nind = m_pCurrRegion->GetNIndividuals();
    const Region *cnst_Reg =  m_pCurrRegion;// jrmhack around c++ inflexibility

    while((pop = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,pop)))
    {
        TiXmlElement * populationElement = pop->ToElement();
        TiXmlNode * ind = NULL;
        while((ind = populationElement->IterateChildren(xmlstr::XML_TAG_INDIVIDUAL,ind)))
        {
            TiXmlElement * individualElement = ind->ToElement();
            string xmlindname = getNodeAttributeValue(individualElement,xmlstr::XML_ATTRTYPE_NAME);
            string token;

            LongVec2d phasesites;

            vector<TiXmlElement *> phaseElements
                = getAllOptionalDescendantElements (individualElement,xmlstr::XML_TAG_PHASE);
            if(phaseElements.size() == 0)
            {
                // no phase info available; we set empty vectors
                //EWFIX.P5 DIMENSION--needs to be 3d?
                LongVec2d phases(m_pCurrRegion->GetNumFixedLoci());
                phasesites = phases;
            }
            else
            {
                TiXmlNode * child = NULL;
                long locus = 0;
                while((child = individualElement->IterateChildren(xmlstr::XML_TAG_PHASE,child)))
                {
                    TiXmlElement * phaseElement = child->ToElement();

                    string content = getNodeText(phaseElement);
                    string phasetype = getNodeAttributeValue(phaseElement,xmlstr::XML_ATTRTYPE_TYPE);
                    StripLeadingSpaces(phasetype);
                    StripTrailingSpaces(phasetype);

                    LongVec1d phases = ProduceLongVec1dOrBarf(getNodeText(phaseElement));
                    long offset = m_pCurrRegion->GetOffset(locus);
                    long length = m_pCurrRegion->GetLocusNsites(locus);
                    for (LongVec1d::iterator phase=phases.begin(); phase != phases.end(); phase++)
                    {
                        *phase -= offset;
                        if (*phase < 0)
                        {
                            m_parser.ThrowDataError("One or more phase values are lower than " + ToString(offset) + ", the offset for locus " + ToString(locus+1) + ".", phaseElement->Row());
                        }
                        if (*phase >= length)
                        {
                            m_parser.ThrowDataError("One or more phase values are higher than " + ToString(length + offset - 1) + ", the offset plus the length of locus " + ToString(locus+1) + ".", phaseElement->Row());
                        }
                    }
                    std::sort(phases.begin(), phases.end());

                    if (phasetype == xmlstr::XML_ATTRVALUE_KNOWN)
                    {
                        //We need to reverse the vector, though only for markers.
                        LongVec1d all_locations = m_pCurrRegion->GetLocus(locus).GetMarkerLocations();
                        for (LongVec1d::iterator knownphase = phases.begin();
                             knownphase != phases.end() ; knownphase++)
                        {
                            LongVec1d::iterator known = FindValIn(*knownphase, all_locations);
                            if (known == all_locations.end())
                            {
                                m_parser.ThrowDataError("Site " + ToString(*knownphase + offset) + " in locus " + ToString(locus+1) + " does not have a marker associated with it, and may therefore not be set 'phase known'.", phaseElement->Row());
                            }
                            all_locations.erase(known);
                        }
                        phases = all_locations;
                    }
                    else if (!(phasetype == xmlstr::XML_ATTRVALUE_UNKNOWN))
                    {
                        m_parser.ThrowDataError("Unknown type of phase information " + phasetype, phaseElement->Row());
                    }
                    phasesites.push_back(phases);
                    ++locus;
                }
            }

            // Error check for consistency
            if (phasesites.size() != static_cast<unsigned long>(m_pCurrRegion->GetNumFixedLoci()))
            {
                m_parser.ThrowDataError("Number of phase entries inconsistent with number of segments",individualElement->Row());
            }

            // Note: the following is a hack to get panels working
            // without a major refactoring of ParseTreeToData

            // find individual and set phase markers
            for (int i=0; i<nind;i++)
            {
                string localindname = (cnst_Reg->GetIndividual(i)).GetName();
                //if(CompareWOCase(localindname,xmlindname))
                if(xmlindname.compare(localindname) == 0)
                {
                    try
                    {
                        m_pCurrRegion->SetPhaseMarkers(i, phasesites);
                    }
                    catch (const data_error& e)
                    {
                        m_parser.ThrowDataError(e.what(), individualElement->Row());
                    }
                }
            }
            indnum++;
        }

        // now find all the panels (assuming they are everything
        // that does not have phasesites set) and set their phase markers
        for (int i=0; i<nind;i++)
        {
            string indname = (cnst_Reg->GetIndividual(i)).GetName();
            if(cnst_Reg->GetIndividual(i).GetPhaseSites().empty())
                //            vector<Branch_ptr> tipvec= (cnst_Reg->GetIndividual(i)).GetAllTips();
                //            int numtips = tipvec.size();
                // all tips should be of the same type so use the first one
                //            if (tipvec[0]->m_isSample == 0) // jrmfix this has to go
            {
                m_pCurrRegion->SetPhaseMarkers(i, panelsites);
            }
        }
    }
} // DoPhases

//------------------------------------------------------------------------------------

UserTree*
ParseTreeToData::DoTree(TiXmlElement * treeElement)
{
    // WARNING:  This should eventually be done with a TreeFactory.
    return new NewickTree(getNodeText(treeElement));
} // DoTree

//------------------------------------------------------------------------------------

void
ParseTreeToData::DoEffectivePopSize(TiXmlElement * sizeElement)
{
    // default value if sizeElement is Null
    double size = defaults::effpopsize;

    if (sizeElement != NULL)
    {
        try
        {
            size = ProduceDoubleOrBarf(getNodeText(sizeElement));
        }
        catch (const data_error& e)
        {
            m_parser.ThrowDataError(string(e.what()),sizeElement->Row());
        }
    }
    //LS NOTE:  This sets the effective population size in the actual data
    // pack, but in the menu, there is an effective population size menu
    // where the information is stored in ui_vars_datapackplus.  This means
    // that when the datapackplus is created, it must set those values properly
    // from what we read in here.  This all works for now, but is a bit arcane.
    // If we move to having a front-end datapack as well as a back-end datapack,
    // this would theoretically become a bit simpler.
    if (size <= 0)
    {
        m_parser.ThrowDataError("All effective population sizes must be positive.");
    }
    m_pCurrRegion->SetEffectivePopSize(size);

} // DoEffectivePopSize

//____________________________________________________________________________________

LongVec1d::iterator FindValIn(long val, LongVec1d& vec)
{
    LongVec1d::iterator found = vec.begin();
    for (; found != vec.end(); found++)
    {
        if (*found == val) return found;
    }
    return found;
}

//____________________________________________________________________________________

void
ParseTreeToData::DiagnoseTagForMigrationalForce(TiXmlElement * rootElement)
{
    m_migrationalForce = force_MIG;

    TiXmlElement * forcesElement = singleOptionalChild(rootElement,xmlstr::XML_TAG_FORCES);
    if (forcesElement == NULL) return;

    TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_DIVERGENCE);
    if (forceElement == NULL) return;

    m_migrationalForce = force_DIVMIG;
    return;
}

//____________________________________________________________________________________

force_type
ParseTreeToData::TagForMigrationalForce() const
{
    assert(m_migrationalForce != force_NONE);
    return m_migrationalForce;
}

//____________________________________________________________________________________

bool
ParseTreeToData::CheckARGtree(vector<UIId> ids, bool batchmode)
{
    // This is an unspeakably long function, but breaking it into
    // subroutines really doesn't gain anything.
    // I tried to keep it very modular so it's obvious what is going on.
    // That makes it a bit more verbose.
    bool retval = true;
    // makes sure that the ARG tree for a region is still valid
    TiXmlElement* docElement    = m_parser.GetRootElement();
    TiXmlElement* forcesElement = singleRequiredChild(docElement,xmlstr::XML_TAG_FORCES);
    TiXmlElement* dataElement   = singleRequiredChild(docElement,xmlstr::XML_TAG_DATA);
    TiXmlNode * child = NULL;
    while((child = dataElement->IterateChildren(xmlstr::XML_TAG_REGION,child)))
    {
        string nodetype;
        TiXmlElement * regionElement = child->ToElement();
        TiXmlElement * treeElement = singleOptionalChild(regionElement,xmlstr::XML_TAG_TREE);
        if (treeElement != NULL)
        {
            string treetype = getNodeAttributeValue(treeElement,xmlstr::XML_ATTRTYPE_TYPE);
            if (treetype == xmlstr::XML_STRING_ARG)
            {
                // validate node data
                TiXmlElement * graphElement = singleOptionalChild(treeElement,xmlstr::XML_TAG_GRAPH);
                TiXmlNode * node = NULL;
                while((node = graphElement->IterateChildren(xmlstr::XML_TAG_NODE,node)))
                {
                    // pick up node element
                    TiXmlElement * nodeElement = node->ToElement();
                    string id = getNodeAttributeValue(nodeElement,xmlstr::XML_ATTRTYPE_ID);

                    string label;
                    bool typefound = false;
                    bool labelfound = false;
                    bool timefound = false;
                    bool recfound = false;
                    TiXmlNode * argdata = NULL;

                    // check for argdata keys and pick up label data
                    while((argdata = nodeElement->IterateChildren(xmlstr::XML_TAG_ARGDATA,argdata)))
                    {
                        TiXmlElement * argdataElement = argdata->ToElement();
                        string key = getNodeAttributeValue(argdataElement,xmlstr::XML_ATTRTYPE_KEY);
                        if (key == xmlstr::XML_ATTRVALUE_NODE_TYPE)
                        {
                            nodetype = getNodeText(argdataElement);
                            typefound = true;
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_NODE_LABEL)
                        {
                            label = getNodeText(argdataElement);
                            labelfound = true;
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_NODE_TIME)
                        {
                            timefound = true;
                        }
                        else if(key == xmlstr::XML_ATTRVALUE_REC_LOCATION)
                        {
                            recfound = true;
                        }
                        else
                        {
                            // unknown key
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                            error_msg += id;
                            error_msg += " contains unknown key: ";
                            error_msg += key;
                            error_msg += ". ARG tree is not valid.\n\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }

                    // data present checks
                    if (!typefound)
                    {
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                        error_msg += id;
                        error_msg += " has no node_type";
                        error_msg += ". ARG tree is not valid.\n\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    if (!timefound)
                    {
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                        error_msg += id;
                        error_msg += " type: ";
                        error_msg += nodetype;
                        error_msg += " has no node_time";
                        error_msg += ". ARG tree is not valid.\n\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    if (nodetype == xmlstr::XML_BRANCHTYPE_TIP)
                    {
                        if (!labelfound)
                        {
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                            error_msg += id;
                            error_msg += " type: ";
                            error_msg += nodetype;
                            error_msg += " does not contain a node_label";
                            error_msg += ". ARG tree is not valid.\n\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }
                    else if (nodetype == xmlstr::XML_BRANCHTYPE_REC)
                    {
                        if (!recfound)
                        {
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                            error_msg += id;
                            error_msg += " type: ";
                            error_msg += nodetype;
                            error_msg += " does not contain a rec_location";
                            error_msg += ". ARG tree is not valid.\n\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }

                    // see if input tip exists in the population sample name set
                    bool tipfound = false;
                    if (nodetype == xmlstr::XML_BRANCHTYPE_TIP)
                    {
                        // check labels against tips defined in data set
                        TiXmlNode * pop = NULL;
                        while((pop = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,pop)))
                        {
                            TiXmlElement * populationElement = pop->ToElement();
                            TiXmlNode * indiv = NULL;
                            while((indiv = populationElement->IterateChildren(xmlstr::XML_TAG_INDIVIDUAL,indiv)))
                            {
                                TiXmlElement * individualElement = indiv->ToElement();
                                TiXmlNode * samp = NULL;
                                while((samp = individualElement->IterateChildren(xmlstr::XML_TAG_SAMPLE,samp)))
                                {
                                    TiXmlElement * sampleElement = samp->ToElement();
                                    string name = getNodeAttributeValue(sampleElement,xmlstr::XML_ATTRTYPE_NAME);
                                    boost::trim(label);
                                    boost::trim(name);
                                    if (label == name)
                                    {
                                        tipfound = true;
                                    }
                                }
                            }
                        }

                        if (!tipfound)
                        {
                            // write an error message
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree node: ";
                            error_msg += id;
                            error_msg += " Tip: ";
                            error_msg += label;
                            error_msg += " not found in Population Sample names. ARG tree is not valid.\n\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }
                    else
                    {
                        // This is the one place the batch and interactive versions diverge.
                        //
                        // In interactive mode the user can set forces which then need to be
                        // checked against the forces in the ARG tree after they hit execute.
                        // Those are in the ids array.
                        //
                        // In a batch run, the forces can't change but they
                        // must be picked up from the XML because the ids array
                        // doesn't get defined until parsetreetosettings is run,
                        // which happens after parsetreetodata.

                        // check type against allowed forces
                        bool forcefound = true;
                        if (nodetype == xmlstr::XML_BRANCHTYPE_COAL)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_COALESCENCE);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_COAL) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else if (nodetype == xmlstr::XML_BRANCHTYPE_DISEASE)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_DISEASE);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_DISEASE) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else if (nodetype == xmlstr::XML_BRANCHTYPE_DIVMIG)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_DIVMIG);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_DIVMIG) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else if (nodetype == xmlstr::XML_BRANCHTYPE_EPOCH)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_DIVERGENCE);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_DIVERGENCE) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else if (nodetype == xmlstr::XML_BRANCHTYPE_MIG)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_MIGRATION);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_MIG) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else if (nodetype == xmlstr::XML_BRANCHTYPE_REC)
                        {
                            if (batchmode)
                            {
                                TiXmlElement * forceElement = singleOptionalChild(forcesElement,xmlstr::XML_TAG_RECOMBINATION);
                                if(forceElement == NULL)
                                {
                                    forcefound = false;
                                }
                            }
                            else
                            {
                                if(find(ids.begin(), ids.end(), force_REC) == ids.end())
                                {
                                    forcefound = false;
                                }
                            }
                        }
                        else
                        {
                            forcefound = false;
                        }
                        if (!forcefound)
                        {
                            // write an error message
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: force type: ";
                            error_msg += nodetype;
                            error_msg += " is not active. ARG tree is not valid.\n\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }
                }

                // check population samples against the ARG tips
                TiXmlNode * pop = NULL;
                while((pop = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,pop)))
                {
                    TiXmlElement * populationElement = pop->ToElement();
                    TiXmlNode * indiv = NULL;
                    while((indiv = populationElement->IterateChildren(xmlstr::XML_TAG_INDIVIDUAL,indiv)))
                    {
                        TiXmlElement * individualElement = indiv->ToElement();
                        TiXmlNode * samp = NULL;
                        while((samp = individualElement->IterateChildren(xmlstr::XML_TAG_SAMPLE,samp)))
                        {
                            TiXmlElement * sampleElement = samp->ToElement();
                            string name = getNodeAttributeValue(sampleElement,xmlstr::XML_ATTRTYPE_NAME);

                            bool samplefound = false;
                            TiXmlElement * graphElement = singleOptionalChild(treeElement,xmlstr::XML_TAG_GRAPH);
                            TiXmlNode * node = NULL;
                            while((node = graphElement->IterateChildren(xmlstr::XML_TAG_NODE,node)))
                            {
                                // pick up branch element
                                TiXmlElement * nodeElement = node->ToElement();
                                string eletype;
                                string elelabel;
                                TiXmlNode * argdata = NULL;
                                while((argdata = nodeElement->IterateChildren(xmlstr::XML_TAG_ARGDATA,argdata)))
                                {
                                    TiXmlElement * argdataElement = argdata->ToElement();
                                    string key = getNodeAttributeValue(argdataElement,xmlstr::XML_ATTRTYPE_KEY);
                                    if (key == xmlstr::XML_ATTRVALUE_NODE_TYPE)
                                    {
                                        eletype = getNodeText(argdataElement);
                                    }
                                    if (key == xmlstr::XML_ATTRVALUE_NODE_LABEL)
                                    {
                                        elelabel = getNodeText(argdataElement);
                                    }
                                }
                                if (eletype == xmlstr::XML_BRANCHTYPE_TIP)
                                {
                                    boost::trim(elelabel);
                                    boost::trim(name);
                                    if (elelabel == name)
                                    {
                                        samplefound = true;
                                    }
                                }
                            }
                            if (!samplefound)
                            {
                                // write an error message
                                string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Population Sample name: ";
                                error_msg += name;
                                error_msg += " not found in ARG tree Tips. ARG tree is not valid.\n\n";
                                retval = false;
                                printf("%s", error_msg.c_str());
                            }
                        }
                    }
                }

                // check edge information
                TiXmlNode * edge = NULL;
                int nedge = 0;
                while((edge = graphElement->IterateChildren(xmlstr::XML_TAG_EDGE,edge)))
                {
                    // pick up edge element
                    TiXmlElement * edgeElement = edge->ToElement();

                    // check source and target defined
                    string source = getNodeAttributeValue(edgeElement,xmlstr::XML_ATTRTYPE_SOURCE);
                    if (source.empty())
                    {
                        // write an error message
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                        error_msg += nedge;
                        error_msg +=" does not have a source. ";
                        error_msg += " ARG tree is not valid.\n\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    string target = getNodeAttributeValue(edgeElement,xmlstr::XML_ATTRTYPE_TARGET);
                    if (target.empty())
                    {
                        // write an error message
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                        error_msg += nedge;
                        error_msg +=" does not have target. ";
                        error_msg += "ARG tree is not valid.\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    if (source == target)
                    {
                        // write an error message
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                        error_msg += nedge;
                        error_msg +=" source and target same: ";
                        error_msg += source;
                        error_msg += ". ARG tree is not valid.\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    // check argdata
                    bool partfound = false;
                    TiXmlNode * argdata = NULL;
                    while((argdata = edgeElement->IterateChildren(xmlstr::XML_TAG_ARGDATA,argdata)))
                    {
                        TiXmlElement * argdataElement = argdata->ToElement();
                        string key = getNodeAttributeValue(argdataElement,xmlstr::XML_ATTRTYPE_KEY);

                        // check if forces the partitions apply to exist
                        if (key == xmlstr::XML_ATTRVALUE_PARTITIONS)
                        {
                            string partitions = getNodeText(argdataElement);
                            if (partitions.length() >0)
                            {
                                unsigned int idx = partitions.find(',');
                                unsigned int stidx = 0;
                                do
                                {
                                    // make sure forces are still defined
                                    string psubstr = partitions.substr(stidx, idx-stidx);
                                    unsigned int jdx = psubstr.find(':');
                                    if (jdx == psubstr.length())
                                    {
                                        // write an error message
                                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                                        error_msg += nedge;
                                        error_msg +=" force: ";
                                        error_msg += psubstr;
                                        error_msg += " does not have a partition. ARG tree is not valid.\n";
                                        retval = false;
                                        printf("%s", error_msg.c_str());
                                    }
                                    else
                                    {
                                        // validate forces
                                        string forcekind = psubstr.substr(0, jdx);
                                        string forcevalue = psubstr.substr(jdx+1, psubstr.length());

                                        bool forceactive = true;
                                        bool popfound = true;
                                        if (forcekind == uistr::disease)
                                        {
                                            if(find(ids.begin(), ids.end(), force_DISEASE) == ids.end())
                                            {
                                                forceactive = false;
                                            }
                                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                                            error_msg += nedge;
                                            error_msg +=" force: ";
                                            error_msg += forcekind;
                                            error_msg += " not yet implemented.";
                                            error_msg += " ARG tree is not valid.\n";
                                            retval = false;
                                            printf("%s", error_msg.c_str());
                                        }
                                        else if (forcekind == uistr::divmigration)
                                        {
                                            if(find(ids.begin(), ids.end(), force_DIVERGENCE) == ids.end())
                                            {
                                                forceactive = false;
                                            }
                                            else
                                            {
                                                popfound = false;
                                                // check population names against the force value
                                                TiXmlNode * pop = NULL;
                                                while((pop = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,pop)))
                                                {
                                                    TiXmlElement * populationElement = pop->ToElement();
                                                    string source =
                                                        getNodeAttributeValue(populationElement,xmlstr::XML_ATTRTYPE_NAME);
                                                    if (source == forcevalue)
                                                    {
                                                        popfound = true;
                                                    }
                                                }

                                                if (!popfound)
                                                {
                                                    // check ancestor names against the force value
                                                    TiXmlElement * forcesElement =
                                                        singleOptionalChild(docElement,xmlstr::XML_TAG_FORCES);
                                                    if(forcesElement != NULL)
                                                    {
                                                        TiXmlElement * forceElement =
                                                            singleOptionalChild(forcesElement,xmlstr::XML_TAG_DIVERGENCE);
                                                        if (forceElement != NULL)
                                                        {
                                                            TiXmlElement * epochsElement =
                                                                singleRequiredChild(forceElement,xmlstr::XML_TAG_POPTREE);
                                                            TiXmlNode * epchild = NULL;
                                                            while((epchild =
                                                                   epochsElement->IterateChildren(xmlstr::XML_TAG_EPOCH_BOUNDARY, epchild)))
                                                            {
                                                                TiXmlElement * boundaryElement = epchild->ToElement();
                                                                TiXmlElement * ancestorElement =
                                                                    singleOptionalChild(boundaryElement,xmlstr::XML_TAG_ANCESTOR);
                                                                string epochname = getNodeText(ancestorElement);
                                                                if (epochname == forcevalue)
                                                                {
                                                                    popfound = true;
                                                                }
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                        else if(forcekind == uistr::migration)
                                        {
                                            if(find(ids.begin(), ids.end(), force_MIG) == ids.end())
                                            {
                                                forceactive = false;
                                            }
                                            else
                                            {
                                                popfound = false;
                                                // check population names against the force value
                                                TiXmlNode * pop = NULL;
                                                while((pop = regionElement->IterateChildren(xmlstr::XML_TAG_POPULATION,pop)))
                                                {
                                                    TiXmlElement * populationElement = pop->ToElement();
                                                    string source = getNodeAttributeValue(populationElement,xmlstr::XML_ATTRTYPE_NAME);
                                                    if (source == forcevalue)
                                                    {
                                                        popfound = true;
                                                    }
                                                }
                                            }
                                        }
                                        else
                                        {
                                            // invalid divergence force
                                            // write an error message
                                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                                            error_msg += nedge;
                                            error_msg +=" force: ";
                                            error_msg += forcekind;
                                            error_msg += " is not a valid divergence force.";
                                            error_msg += " ARG tree is not valid.\n";
                                            retval = false;
                                            printf("%s", error_msg.c_str());
                                        }

                                        if (!forceactive)
                                        {
                                            // write an error message
                                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                                            error_msg += nedge;
                                            error_msg +=" force: ";
                                            error_msg += forcekind;
                                            error_msg += " is not active. ARG tree is not valid.\n";
                                            retval = false;
                                            printf("%s", error_msg.c_str());
                                        }

                                        if (!popfound)
                                        {
                                            // write an error message
                                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: Edge: ";
                                            error_msg += nedge;
                                            error_msg +=" force: ";
                                            error_msg += forcekind;
                                            error_msg +=" applies to population: ";
                                            error_msg += forcevalue;
                                            error_msg += " which does not exist. ARG tree is not valid.\n";
                                            retval = false;
                                            printf("%s", error_msg.c_str());
                                        }
                                    }
                                    stidx = idx + 1;
                                    idx = partitions.find(',', stidx);
                                }
                                while (idx<partitions.length());
                            }
                            else
                            {
                                partfound = true;
                            }
                        }

                        else if (key == xmlstr::XML_ATTRVALUE_TRANS_SITES)
                        {
                            string sites = getNodeText(argdataElement);
                            if (sites.length() <=0)
                            {
                                // no sites
                                string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree Edge: ";
                                error_msg += nedge;
                                error_msg +="  no transmitted sites. ARG tree is not valid.\n";
                                retval = false;
                                printf("%s", error_msg.c_str());
                            }
                        }

                        // something wacky happened
                        else if (key != xmlstr::XML_ATTRVALUE_LIVE_SITES) // live_sites is in here for GraphML analysis
                                                                          // it's not needed by LAMARC
                        {
                            // unknown key
                            string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree Edge: ";
                            error_msg += nedge;
                            error_msg +=" contains unknown key: ";
                            error_msg += key;
                            error_msg += ". ARG tree is not valid.\n";
                            retval = false;
                            printf("%s", error_msg.c_str());
                        }
                    }

                    // make sure source and target exist in the ARG tree
                    bool sourcefound = false;
                    bool targetfound = false;
                    TiXmlElement * graphElement = singleOptionalChild(treeElement,xmlstr::XML_TAG_GRAPH);
                    TiXmlNode * node = NULL;
                    while((node = graphElement->IterateChildren(xmlstr::XML_TAG_NODE,node)))
                    {
                        // pick up node element
                        TiXmlElement * nodeElement = node->ToElement();
                        string id = getNodeAttributeValue(nodeElement,xmlstr::XML_ATTRTYPE_ID);
                        if (source == id)
                        {
                            sourcefound = true;
                        }
                        if (target == id)
                        {
                            targetfound = true;
                        }
                    }

                    if ((!sourcefound) && (source != "1"))
                    {
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree edge source: ";
                        error_msg += source;
                        error_msg += " not a node. ARG tree is not valid.\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    if (!targetfound)
                    {
                        string error_msg = "ERROR found in ParseTreeToData::CheckARGtree: ARG tree edge target: ";
                        error_msg += target;
                        error_msg += " not a node. ARG tree is not valid.\n";
                        retval = false;
                        printf("%s", error_msg.c_str());
                    }

                    nedge++;
                }
            }
        }
    }

    return retval;
} //CheckARGtree

//____________________________________________________________________________________

bool
ParseTreeToData::DoARGtree()
{
    // this takes the xml input data for an ARG tree which is node centric
    // and turns it into a local data structure that is edge centric for Lamarc

    bool retval = true;

    // build the ARG tree for use in ARGTree::ToLamarcTree
    TiXmlElement* docElement = m_parser.GetRootElement();
    TiXmlElement* dataElement = singleRequiredChild(docElement,xmlstr::XML_TAG_DATA);
    TiXmlNode * child = NULL;
    while((child = dataElement->IterateChildren(xmlstr::XML_TAG_REGION,child)))
    {
        TiXmlElement * regionElement = child->ToElement();
        string regionName = getNodeAttributeValue(regionElement,xmlstr::XML_ATTRTYPE_NAME);
        Region* region = datapack.GetRegionByName(regionName);

        TiXmlElement * treeElement = singleOptionalChild(regionElement,xmlstr::XML_TAG_TREE);
        if (treeElement != NULL)
        {
            string treetype = getNodeAttributeValue(treeElement,xmlstr::XML_ATTRTYPE_TYPE);
            if (treetype == xmlstr::XML_STRING_ARG)
            {
                // create edges
                TiXmlElement * graphElement = singleOptionalChild(treeElement,xmlstr::XML_TAG_GRAPH);
                TiXmlNode * edge = NULL;
                while((edge = graphElement->IterateChildren(xmlstr::XML_TAG_EDGE,edge)))
                {
                    // get edge element data
                    TiXmlElement * edgeElement = edge->ToElement();
                    long source = atol(getNodeAttributeValue(edgeElement,xmlstr::XML_ATTRTYPE_SOURCE).c_str());
                    long target = atol(getNodeAttributeValue(edgeElement,xmlstr::XML_ATTRTYPE_TARGET).c_str());
                    ARGEdge argedge(target, source);

                    string partitions = "";
                    string livesites = "";
                    string transmittedsites = "";
                    TiXmlNode * argdata = NULL;
                    while((argdata = edgeElement->IterateChildren(xmlstr::XML_TAG_ARGDATA,argdata)))
                    {
                        TiXmlElement * argdataElement = argdata->ToElement();
                        string key = getNodeAttributeValue(argdataElement,xmlstr::XML_ATTRTYPE_KEY);
                        if (key == xmlstr::XML_ATTRVALUE_PARTITIONS)
                        {
                            partitions = getNodeText(argdataElement);
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_LIVE_SITES)
                        {
                            livesites = getGraphMLNodeText(argdataElement); // fix GraphML tweaks
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_TRANS_SITES)
                        {
                            transmittedsites = getGraphMLNodeText(argdataElement); // fix GraphML tweaks
                        }
                    }
                    argedge.SetPartitions(partitions);
                    argedge.SetLiveSites(livesites);  
                    argedge.SetTransmittedSites(transmittedsites); 
                    region->m_argedges.push_back(argedge);
                }

                // read nodes and hook them to the edges
                TiXmlNode * node = NULL;
                while((node = graphElement->IterateChildren(xmlstr::XML_TAG_NODE,node)))
                {
                    // get node element data
                    TiXmlElement * nodeElement = node->ToElement();
                    long nodeid = atol(getNodeAttributeValue(nodeElement,xmlstr::XML_ATTRTYPE_ID).c_str());
                    string type = "";
                    string label = "";
                    double time = FLAGDOUBLE;
                    long recloc = FLAGLONG;
                    TiXmlNode * data = NULL;
                    while((data = nodeElement->IterateChildren(xmlstr::XML_TAG_ARGDATA,data)))
                    {
                        TiXmlElement * dataElement = data->ToElement();
                        string key = getNodeAttributeValue(dataElement,xmlstr::XML_ATTRTYPE_KEY);
                        if (key == xmlstr::XML_ATTRVALUE_NODE_TYPE)
                        {
                            type = getNodeText(dataElement);
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_NODE_LABEL)
                        {
                            label = getNodeText(dataElement);
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_NODE_TIME)
                        {
                            time = ProduceDoubleOrBarf(getNodeText(dataElement));
                        }
                        else if (key == xmlstr::XML_ATTRVALUE_REC_LOCATION)
                        {
                            recloc = atol(getNodeText(dataElement).c_str()) - 2; // undo GraphML tweak
                        }
                    }

                    // find the edge that points at this node
                    for(size_t edge=0; edge<region->m_argedges.size(); edge++)
                    {
                        if (region->m_argedges[edge].GetTarget() == nodeid)
                        {
                            region->m_argedges[edge].SetTargetId(nodeid);
                            region->m_argedges[edge].SetType(type);
                            region->m_argedges[edge].SetTime(time);
                            region->m_argedges[edge].SetLabel(label);
                            region->m_argedges[edge].SetRecLoc(recloc);
                        }
                    }
                }

#ifndef NDEBUG
                // debug print of final edge list to see what happened
                printf("\n****Final edge list at end of DoARGtree****\n");
                for(size_t edge=0; edge<region->m_argedges.size(); edge++)
                {
                    printf("\nedge: %i  target: %i source: %i\n", (int)edge, (int)(region->m_argedges[edge].GetTarget()), (int)region->m_argedges[edge].GetSource());
                    printf("Partitions: %s\n", region->m_argedges[edge].GetPartitions().c_str());
                    printf("Live sites: %s\n",  region->m_argedges[edge].GetLiveSites().c_str());
                    printf("Transmitted sites: %s\n",  region->m_argedges[edge].GetTransmittedSites().c_str());
                    printf("Target Id: %i Label: %s\n", (int)region->m_argedges[edge].GetTargetId(), region->m_argedges[edge].GetLabel().c_str());
                    printf("Type: %s  Time: %f\n", region->m_argedges[edge].GetType().c_str(), region->m_argedges[edge].GetTime());
                    printf("Rec Loc: %i\n", (int)(region->m_argedges[edge].GetRecLoc()));
                }
#endif
            }
        }
    }

    return retval;
} //DoARGtree

//____________________________________________________________________________________
