// $Id: gc_datastore.cpp,v 1.105 2018/01/03 21:32:55 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Mary Kuhner, Jon Yamato, and Joseph Felsenstein */


#include <cassert>

#include "cmdfileschema.h"
#include "cnv_strings.h"
#include "errhandling.h"
#include "front_end_warnings.h"     // EWFIX -- try to get rid of this
#include "gc_data.h"
#include "gc_datastore.h"
#include "gc_errhandling.h"
#include "gc_data.h"
#include "gc_default.h"
#include "gc_file.h"
#include "gc_file_util.h"
#include "gc_infile_err.h"
#include "gc_loci_match.h"
#include "gc_locus.h"
#include "gc_map_err.h"
#include "gc_migrate.h"
#include "gc_parse.h"
#include "gc_parse_block.h"
#include "gc_parse_locus.h"
#include "gc_parse_sample.h"
#include "gc_parse_pop.h"
#include "gc_parser.h"
#include "gc_phylip.h"
#include "gc_population.h"
#include "gc_pop_match.h"
#include "gc_region.h"
#include "gc_strings.h"
#include "gc_strings_infile.h"
#include "gc_strings_map.h"
#include "gc_strings_parse.h"
#include "gc_strings_phase.h"
#include "gc_types.h"
#include "parsetreewalker.h"
#include "tixml_util.h"
#include "xml.h"
#include "xml_strings.h"

#include "Converter_DataSourceException.h"

#include "wx/file.h"
#include "wx/filename.h"
#include "wx/log.h"
#include "wx/tokenzr.h"
#include "wx/txtstrm.h"
#include "wx/wfstream.h"
#include "wx/utils.h"

//------------------------------------------------------------------------------------

GCDataStore::GCDataStore()
    :
    m_commandFileCurrentlyBeingParsed(wxEmptyString),
    m_commentString(wxEmptyString),
    m_outfileName(gcstr::exportFileDefault),
    m_structures(this)
{
}

GCDataStore::~GCDataStore()
// DON'T do a thourough delete here -- do it in NukeContents()
{
}

void
GCDataStore::NukeContents()
{
    for(dataFileSet::iterator i=m_dataFiles.begin(); i!=m_dataFiles.end();i++)
    {
        delete *i;
    }
    // EWFIX.LEAK -- do we need to free memory in hap file stuff
}

void
GCDataStore::GCFatal(wxString msg) const
{
    GCError(msg);
    gc_fatal_error e;
    throw e;
}

void
GCDataStore::GCFatalBatchWarnGUI(wxString msg) const
{
    GCFatal(msg);
}

void
GCDataStore::GCFatalUnlessDebug(wxString msg) const
{
#ifdef NDEBUG
    GCFatal(msg);
#else
    GCError(msg);
#endif
}

void
GCDataStore::GCError(wxString msg) const
{
    wxLogError(msg);
}

void
GCDataStore::GCInfo(wxString msg) const
{
    wxLogInfo(msg);
}

void
GCDataStore::GCWarning(wxString msg) const
{
    wxLogWarning(msg);
}

void
GCDataStore::GettingBusy(const wxString& msg) const
{
    wxLogMessage(msg);
}

void
GCDataStore::LessBusy(const wxString& msg) const
{
    wxLogMessage(msg);
}

void
GCDataStore::fileRejectingError(wxString msg,size_t lineNo) const
{
    gc_infile_err e(msg);
    e.setRow(lineNo);
    throw e;
}

void
GCDataStore::batchFileRejectGuiLog(wxString msg, size_t lineNo) const
{
    fileRejectingError(msg,lineNo);
}

void
GCDataStore::warnLog(wxString msg) const
{
    wxLogMessage(msg);
}

void
GCDataStore::warnLog(wxString msg, size_t lineNo) const
{
    wxLogMessage(gcstr::nearRow,(int)lineNo,msg.c_str());
}

bool
GCDataStore::guiQuestionBatchLog(wxString msg, wxString stopButton, wxString continueButton) const
{
    warnLog(msg);
    return true;    // tells caller to continue on (false -> throw)
}

GCParse *
GCDataStore::OneParse(GCFile &              file,
                      GCFileFormat        format,
                      gcGeneralDataType   dataType,
                      GCInterleaving      interleaving)
{
    if(format == format_PHYLIP)
    {
        GCPhylipParser phy(*this);
        GCParse * parse = phy.Parse(file,dataType,interleaving);
        return parse;
    }

    if(format == format_MIGRATE)
    {
        GCMigrateParser mig(*this);
        GCParse * parse = mig.Parse(file,dataType,interleaving);
        return parse;
    }
    return NULL;

}

GCParseVec
GCDataStore::GoodPhylipParses(GCFile & file)
{
    GCParseVec goodParses;

    /////////// parse phylip sequential
    try
    {
        GCPhylipParser phy(*this);
        GCParse * phyParseC = phy.Parse(file,gcdata::nucDataTypes(),interleaving_SEQUENTIAL);
        goodParses.push_back(phyParseC);
    }
    catch(const gc_ex& e)
        // do nothing, we're just avoiding putting parse into return vec
    {
        wxLogVerbose(wxString::Format(gcerr_infile::unableToParseBecause,
                                      file.GetName().c_str(),
                                      ToWxString(format_PHYLIP).c_str(),
                                      ToWxString(gcdata::nucDataTypes()).c_str(),
                                      ToWxString(interleaving_SEQUENTIAL).c_str(),
                                      e.what()));
    }

    /////////// parse phylip interleaved
    try
    {
        GCPhylipParser phy(*this);
        GCParse * phyParseI = phy.Parse(file,gcdata::nucDataTypes(),interleaving_INTERLEAVED);
        goodParses.push_back(phyParseI);
    }
    catch(const gc_ex& e)
        // do nothing, we're just avoiding putting parse into return vec
    {
        wxLogVerbose(wxString::Format(gcerr_infile::unableToParseBecause,
                                      file.GetName().c_str(),
                                      ToWxString(format_PHYLIP).c_str(),
                                      ToWxString(gcdata::nucDataTypes()).c_str(),
                                      ToWxString(interleaving_INTERLEAVED).c_str(),
                                      e.what()));
    }

    goodParses.MungeParses();
    return goodParses;
}

GCParseVec
GCDataStore::GoodMigrateParses(GCFile & file, gcGeneralDataType dataType)
{
    GCParseVec goodParses;

    /////////// parse migrate sequential
    try
    {
        GCMigrateParser mig(*this);
        GCParse * migParseC = mig.Parse(file,dataType,interleaving_SEQUENTIAL);
        goodParses.push_back(migParseC);
    }
    catch(const gc_ex& e)
        // do nothing, we're just avoiding putting parse into return vec
    {
        wxLogVerbose(wxString::Format(gcerr_infile::unableToParseBecause,
                                      file.GetName().c_str(),
                                      ToWxString(format_MIGRATE).c_str(),
                                      ToWxString(dataType).c_str(),
                                      ToWxString(interleaving_SEQUENTIAL).c_str(),
                                      e.what()));
    }

    /////////// parse migrate interleaved
    try
    {
        GCMigrateParser mig(*this);
        GCParse * migParseI = mig.Parse(file,dataType,interleaving_INTERLEAVED);
        goodParses.push_back(migParseI);
    }
    catch(const gc_ex& e)
        // do nothing, we're just avoiding putting parse into return vec
    {
        wxLogVerbose(wxString::Format(gcerr_infile::unableToParseBecause,
                                      file.GetName().c_str(),
                                      ToWxString(format_MIGRATE).c_str(),
                                      ToWxString(dataType).c_str(),
                                      ToWxString(interleaving_INTERLEAVED).c_str(),
                                      e.what()));
    }

    goodParses.MungeParses();
    return goodParses;
}

GCParseVec *
GCDataStore::AllParsesForFile(GCFile & file)
{
    GettingBusy(wxString::Format(gcstr_parse::parsingStarting,file.GetShortName().c_str()));
    GCParseVec * parseVecP = new GCParseVec();

    GCParseVec phyParses = GoodPhylipParses(file);
    parseVecP->insert(parseVecP->end(),phyParses.begin(),phyParses.end());

    GCParseVec migParsesKallele = GoodMigrateParses(file,gcdata::allelicDataTypes());
    parseVecP->insert(parseVecP->end(),migParsesKallele.begin(),migParsesKallele.end());

    GCParseVec migParsesNuc = GoodMigrateParses(file,gcdata::nucDataTypes());
    parseVecP->insert(parseVecP->end(),migParsesNuc.begin(),migParsesNuc.end());

    LessBusy(wxString::Format(gcstr_parse::parsingDone,file.GetShortName().c_str()));

    return parseVecP;
}

void
GCDataStore::SetLamarcCommentString(wxString commentString)
{
    m_commentString = commentString;
}

wxString
GCDataStore::GetOutfileName() const
{
    return m_outfileName;
}

void
GCDataStore::SetOutfileName(wxString outfileName)
{
    m_outfileName = outfileName;
}

const GCStructures &
GCDataStore::GetStructures() const
{
    return m_structures;
}

GCStructures &
GCDataStore::GetStructures()
{
    return m_structures;
}

void
GCDataStore::DebugDump(wxString prefix) const
{
    wxLogDebug("%s************************",prefix.c_str());    // EWDUMPOK
    wxLogDebug("%sGC DataStore: DEBUG DUMP",prefix.c_str());    // EWDUMPOK

    // data file contents
    wxLogDebug("%sData Files",(prefix+gcstr::indent).c_str());  // EWDUMPOK
    dataFileSet::iterator fileIter;
    for(fileIter=m_dataFiles.begin(); fileIter != m_dataFiles.end(); fileIter++)
    {
        GCFile *  file = (*fileIter);
        file->DebugDump(prefix+gcstr::indent+gcstr::indent);
    }

    m_structures.DebugDump(prefix+gcstr::indent);

    wxLogDebug("%sPhase Info",(prefix+gcstr::indent).c_str());  // EWDUMPOK

    gcPhaseInfo * info = BuildPhaseInfo(false);
    // false argument allows us to build the object even if
    // there are inconsistencies. this is appropriate since
    // we're doing a debug dump for information

    info->DebugDump(prefix+gcstr::indent+gcstr::indent);
    delete info;

    GCQuantum::ReportMax();
}

gcPhaseInfo *
GCDataStore::BuildPhaseInfo(bool carpIfBroken) const
{
    gcPhaseInfo * phaseInfo = new gcPhaseInfo();

    phaseInfo->AddRecords(m_phaseInfo);

    for(dataFileSet::const_iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        // get data file
        const GCFile & fileRef = **i;

        // get data file parse
        if(GetStructures().HasParse(fileRef))
        {
            const GCParse & parse = GetStructures().GetParse(fileRef);

            if(GetStructures().HasHapFileAdjacent(fileRef.GetId()))
            {
                size_t adjCount = GetStructures().GetHapFileAdjacent(fileRef.GetId());
                gcPhaseInfo * adjRecs = parse.GetPhaseRecordsForAdjacency(adjCount);
                phaseInfo->AddRecords(*adjRecs);
                delete adjRecs;
            }
            else
            {
                // EWFIX.P4 -- can we cut down on the creation and deletion ?
                gcPhaseInfo * defaultRecs = parse.GetDefaultPhaseRecords();
                phaseInfo->AddRecords(*defaultRecs);
                delete defaultRecs;
            }
        }
        else
        {
            if(carpIfBroken)
            {
                gc_parse_missing_err e(fileRef.GetName());
                throw e;
            }
            else
            {
                // EWFIX -- this code is intended to be triggered during
                // a debug dump. If we wanted we could consider augmenting
                // gcPhaseInfo to indicate the missing parse in its
                // DebugDump routine.
            }
        }
    }

    return phaseInfo;
}

#if 0

void
GCDataStore::AddMapFile(wxString mapFileName)
{
    try
    {
        AddMapFileAsXml(mapFileName);
    }
    catch (const tixml_error& f)
    {
        wxLogVerbose(gcstr_map::notXmlMapFileTryOldFmt,
                     wxFileName(mapFileName).GetFullName().c_str(),
                     f.what());
        AddMapFileAsOldFormat(mapFileName);
    }
}

void
GCDataStore::AddMapFileAsXml(wxString mapFileName)
{
    CmdFileSchema schema;
    FrontEndWarnings warnings;
    XmlParser parser(schema,warnings);

    parser.ParseFileData(mapFileName.c_str());
    TiXmlElement * topElem = parser.GetRootElement();

    const char * value = topElem->Value();
    std::string topTag(value);
    bool matches = CaselessStrCmp(cnvstr::TAG_REGIONS,topTag);
    if(!matches)
    {
        gc_map_err e((wxString::Format(gcerr_map::ERR_BAD_TOP_TAG,cnvstr::TAG_REGIONS.c_str(),topTag.c_str())).c_str());
        throw e;
    }

    cmdParseRegions       (topElem,true); // true value allows using existing regions/loci

    std::vector<std::string> warningStrings = warnings.GetAndClearWarnings();
    assert(warningStrings.empty());
}

void
GCDataStore::AddMapFileAsOldFormat(wxString mapFileName)
{
    wxString defaultRegionName = wxFileName(mapFileName).GetFullName();
    ////////////////////////////////////////////////////
    // open the file stream
    if(! ::wxFileExists(mapFileName))
    {
        throw gc_map_file_missing(mapFileName);
    }
    wxFileInputStream fileStream(mapFileName);
    if(!fileStream.Ok())
    {
        throw gc_map_file_read_err(mapFileName);
    }
    // treat it as text
    wxTextInputStream textStream(fileStream);

    ////////////////////////////////////////////////////
    bool atLeastOneLine = false;
    try
    {
        while(true)
        {
            wxString line = ReadLineSafely(&fileStream,&textStream);
            wxStringTokenizer tokens(line);
            wxStringTokenizer tokenizer(line);
            if(tokenizer.HasMoreTokens())
                // if not, a blank line, so we skip it
            {
                atLeastOneLine = true;
                wxString locusName = tokenizer.GetNextToken();
                wxString locations = tokenizer.GetString();
                if(m_structures.HasLocus(locusName))
                {
                    gcLocus & locus = m_structures.GetLocus(locusName);
                    locus.SetLocations(locations);
                }
                else
                {
                    gcRegion & region = m_structures.FetchOrMakeRegion(defaultRegionName);
                    gcLocus & locus = m_structures.MakeLocus(region,locusName,true);
                    locus.SetLocations(locations);
                }
            }
        }
    }
    catch(const gc_eof& e)
    {
        if(!atLeastOneLine)
        {
            throw gc_map_file_empty(mapFileName);
        }
    }
}

#endif

gcTraitInfo &
GCDataStore::AddNewTrait(wxString name)
{
    return m_structures.MakeTrait(name);
}

void
GCDataStore::Rename(GCQuantum & object, wxString newName)
{
    m_structures.Rename(object,newName);
}

#if 0

void
GCDataStore::AddHapFile(wxString hapFileName)
{
    CmdFileSchema schema;
    FrontEndWarnings warnings;
    XmlParser parser(schema,warnings);

    parser.ParseFileData(hapFileName.c_str());
    TiXmlElement * topElem = parser.GetRootElement();

    const char * value = topElem->Value();
    std::string topTag(value);
    bool matches = CaselessStrCmp(cnvstr::TAG_INDIVIDUALS,topTag);
    if(!matches)
    {
        gc_phase_err e((wxString::Format(gcerr_phase::badTopTag,cnvstr::TAG_INDIVIDUALS.c_str(),topTag.c_str())).c_str());
        throw e;
    }

    cmdParseIndividuals       (topElem,hapFileName);

    std::vector<std::string> warningStrings = warnings.GetAndClearWarnings();
    assert(warningStrings.empty());
}

#endif

void
GCDataStore::DiagnosePhaseInfoProblems(const gcPhaseInfo& phaseInfo) const
{
    constObjVector pops     = m_structures.GetConstDisplayablePops();
    constObjVector regions  = m_structures.GetConstDisplayableRegions();

    for(constObjVector::const_iterator pIter = pops.begin(); pIter != pops.end(); pIter++)
    {
        for(constObjVector::const_iterator rIter = regions.begin(); rIter != regions.end(); rIter++)
        {
            // EWFIX.P4.BUG.564 -- make fancier to keep more info
            std::set<wxString> fullSpecAsSample;
            std::set<wxString> fullSpecAsIndividual;
            std::set<wxString> partSpecAsSample;
            std::set<wxString> partSpecAsIndividual;
            std::set<wxString> simpleLabelNoSpec;

            constObjVector loci = m_structures.GetConstDisplayableLinkedLociInMapOrderFor((*rIter)->GetId());
            for(constObjVector::const_iterator lIter=loci.begin(); lIter != loci.end(); lIter++)
            {
                gcIdSet blockIds = m_structures.GetBlockIds((*pIter)->GetId(),(*lIter)->GetId());
                for(gcIdSet::const_iterator b = blockIds.begin(); b != blockIds.end(); b++)
                {
                    const GCParseBlock * blockP = GetParseBlock(*b);
                    const GCParseSamples & samples = blockP->GetSamples();
                    for(size_t s = 0; s < samples.size(); s++)
                    {
                        const GCParseSample & sample = *(samples[s]);

                        wxString name = sample.GetLabel();

                        if(phaseInfo.HasIndividualRecord(name))
                        {
                            assert(!phaseInfo.HasSampleRecord(name));
                            const gcPhaseRecord & rec = phaseInfo.GetIndividualRecord(name);
                            if(rec.HasSamples())
                            {
                                fullSpecAsIndividual.insert(name);
                            }
                            else
                            {
                                partSpecAsIndividual.insert(name);
                            }
                        }

                        if(phaseInfo.HasSampleRecord(name))
                        {
                            assert(!phaseInfo.HasIndividualRecord(name));
                            const gcPhaseRecord & rec = phaseInfo.GetSampleRecord(name);
                            if(rec.HasIndividual())
                            {
                                fullSpecAsSample.insert(name);
                            }
                            else
                            {
                                partSpecAsSample.insert(name);
                            }
                        }

                        if(!phaseInfo.HasSampleRecord(name) && !phaseInfo.HasIndividualRecord(name))
                        {
                            simpleLabelNoSpec.insert(name);
                        }
                    }
                }
            }

            if(!partSpecAsSample.empty() &&  !partSpecAsIndividual.empty())
                // this only works if all
            {
                wxString repSample = *partSpecAsSample.begin();
                wxString repIndividual = *partSpecAsIndividual.begin();
                wxString regionName = (*rIter)->GetName();
                throw gc_phase_matching_confusion(regionName,repSample,repIndividual);
            }
            if(!partSpecAsSample.empty() &&  !simpleLabelNoSpec.empty())
                // this only works if all
            {
                wxString repSample = *partSpecAsSample.begin();
                wxString repSimple = *simpleLabelNoSpec.begin();
                wxString regionName = (*rIter)->GetName();
                throw gc_phase_matching_confusion(regionName,repSample,repSimple);
            }
            if(!partSpecAsIndividual.empty() &&  !simpleLabelNoSpec.empty())
                // this only works if all
            {
                wxString repIndividual = *partSpecAsIndividual.begin();
                wxString repSimple = *simpleLabelNoSpec.begin();
                wxString regionName = (*rIter)->GetName();
                throw gc_phase_matching_confusion(regionName,repIndividual,repSimple);
            }
        }
    }
}

bool
GCDataStore::PhaseInfoHasAnyZeroes() const
{
    return m_phaseInfo.HasAnyZeroes();
}

//____________________________________________________________________________________
