// $Id: gc_datastore_files.cpp,v 1.37 2018/01/03 21:32:55 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Mary Kuhner, Jon Yamato, and Joseph Felsenstein */


#include <cassert>

#include "gc_creation_info.h"
#include "gc_data.h"
#include "gc_datastore.h"
#include "gc_file.h"
#include "gc_infile_err.h"
#include "gc_loci_match.h"
#include "gc_locus_err.h"
#include "gc_parse_block.h"
#include "gc_parse_locus.h"
#include "gc_parse_sample.h"
#include "gc_pop_match.h"
#include "gc_strings.h"
#include "gc_structures_err.h"

#include "wx/file.h"
#include "wx/filename.h"
#include "wx/log.h"

//------------------------------------------------------------------------------------

GCFile &
GCDataStore::AddDataFile(wxString           fullPathFileName,
                         GCFileFormat    format,
                         gcGeneralDataType   dataType,
                         GCInterleaving  interleaving)
// version to use when reading from batch command file
{
    // test for readability
    if(! ::wxFileExists(fullPathFileName))
    {
        throw missing_file_error(fullPathFileName.c_str());
    }

    GCFile * file = new GCFile(*this,fullPathFileName);
    GCParseVec * parseVecP = new GCParseVec();

    try
    {
        GCParse * parse = OneParse(*file,format,dataType,interleaving);
        parseVecP->push_back(parse);
    }
    catch(const gc_ex& e)
    {
        parseVecP->NukeContents();
        delete parseVecP;
        delete file;
        throw;
    }

    file->SetParses(parseVecP);
    m_structures.AddFile(*file);

    m_dataFiles.insert(file);
    wxLogVerbose(gcverbose::addedFile,fullPathFileName.c_str());
    return *file;
}

GCFile &
GCDataStore::AddDataFile(wxString fullPathFileName)
{
    // test for readability
    if(! ::wxFileExists(fullPathFileName))
    {
        throw missing_file_error (fullPathFileName.c_str());
    }

    GCFile * file = new GCFile(*this,fullPathFileName);

    GCParseVec * parseVecP = AllParsesForFile(*file);
    assert(parseVecP != NULL);
    if (parseVecP->size() == 0)
    {
        delete parseVecP;
        delete file;
        throw unparsable_file_error(fullPathFileName.c_str());
    }

    file->SetParses(parseVecP);
    m_structures.AddFile(*file);
    m_dataFiles.insert(file);
    wxLogVerbose(gcverbose::addedFile,fullPathFileName.c_str());

    try
    {
        if(file->GetParseCount() == 1)
        {
            GCPopMatcher p(popmatch_DEFAULT);
            GCLocusMatcher l(locmatch_DEFAULT);
            SetParseChoice(file->GetParse(0),p,l);
        }
    }
    catch (const duplicate_file_error& e)
    {
        GCWarning(e.what());
    }
    catch (const missing_file_error& e)
    {
        GCWarning(e.what());
    }
    catch (const unparsable_file_error& e)
    {
        GCWarning(e.what());
    }

    return *file;
}

const GCParseBlock *
GCDataStore::GetParseBlock(size_t blockId) const
{
    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                if(blockId == blockP->GetId())
                {
                    return blockP;
                }
            }
        }

    }
    assert(false);
    return NULL;
}

constBlockVector
GCDataStore::GetBlocks(size_t popId, size_t locusId) const
{
    gcIdSet ids = m_structures.GetBlockIds(popId,locusId);
    constBlockVector retVal;

    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                size_t blockId = blockP->GetId();
                if(ids.find(blockId) != ids.end())
                {
                    retVal.push_back(blockP);
                }
            }
        }

    }
    return retVal;
}

constBlockVector
GCDataStore::GetBlocksForLocus(size_t locusId) const
{
    gcIdSet ids = m_structures.GetBlocksForLocus(locusId);
    constBlockVector retVal;

    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                size_t blockId = blockP->GetId();
                if(ids.find(blockId) != ids.end())
                {
                    retVal.push_back(blockP);
                }
            }
        }

    }
    return retVal;
}
size_t
GCDataStore::GetDataFileCount() const
{
    return m_dataFiles.size();
}

GCFile &
GCDataStore::GetDataFile(size_t fileIndex)
{
    for(dataFileSet::iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        GCFile & dataFile = **i;
        if(dataFile.GetId() == fileIndex)
        {
            return dataFile;
        }
    }
    wxString msg = wxString::Format(gcerr::missingFileId,(int)fileIndex);
    throw gc_implementation_error (msg.c_str());
}

const GCFile &
GCDataStore::GetDataFile(size_t fileIndex) const
{
    for(dataFileSet::const_iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        const GCFile & dataFile = **i;
        if(dataFile.GetId() == fileIndex)
        {
            return dataFile;
        }
    }
    wxString msg = wxString::Format(gcerr::missingFileId,(int)fileIndex);
    throw gc_implementation_error (msg.c_str());
}

const GCParse &
GCDataStore::GetParse(size_t parseIndex) const
{
    for(dataFileSet::const_iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        const GCFile & dataFile = **i;
        size_t numParses = dataFile.GetParseCount();
        for(size_t p=0; p < numParses; p++)
        {
            const GCParse & parseRef = dataFile.GetParse(p);
            if(parseRef.GetId() == parseIndex)
            {
                return parseRef;
            }
        }
    }
    wxString msg = wxString::Format(gcerr::missingParseId,(int)parseIndex);
    throw gc_implementation_error(msg.c_str());
}

const dataFileSet &
GCDataStore::GetDataFiles() const
{
    return m_dataFiles;
}

size_t
GCDataStore::GetSelectedDataFileCount() const
{
    return m_structures.SelectedFileCount();
}

bool
GCDataStore::HasNoDataFiles() const
{
    return m_dataFiles.empty();
}

bool
GCDataStore::HasUnparsedFiles() const
{
    return m_structures.HasUnparsedFiles();
}

void
GCDataStore::RemoveDataFile(GCFile & fileRef)
{
    dataFileSet::iterator matching = m_dataFiles.find(&fileRef);
    if(matching != m_dataFiles.end())
    {
        wxLogVerbose(gcverbose::removedFile,fileRef.GetName().c_str());
        gcIdSet blocks = fileRef.IdsOfAllBlocks();
        m_structures.RemoveBlocks(blocks);
        m_dataFiles.erase(matching);
        delete &fileRef;
        // EWFIX.P3 -- need an ignore method?? causes UNDO/REDO problem
    }
}

void
GCDataStore::RemoveFiles(bool selectedFilesOnly)
{
    for(dataFileSet::iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        GCFile & fileRef = **iter;
        size_t fileId = fileRef.GetId();
        if(!selectedFilesOnly || m_structures.GetFileSelection(fileId))
        {
            m_structures.RemoveFile(fileId);
            RemoveDataFile(fileRef);
        }

    }
}

void
GCDataStore::SelectAllFiles()
{
    m_structures.AllFileSelectionsTo(true);
}

void
GCDataStore::UnselectAllFiles()
{
    m_structures.AllFileSelectionsTo(false);
}

bool
GCDataStore::GetSelected(const GCFile & fileRef) const
{
    return m_structures.GetFileSelection(fileRef.GetId());
}

void
GCDataStore::SetSelected(const GCFile & fileRef, bool selected)
{
    m_structures.SetFileSelection(fileRef.GetId(),selected);
}

locVector
GCDataStore::GetLociFor(const GCParse & parseRef, const GCLocusMatcher & locMatch)
{
    locVector loci;

    for(size_t i=0; i < parseRef.GetLociCount(); i++)
    {
        GCLocusSpec spec = locMatch.GetLocSpec(i,parseRef);
        try
        {
            gcLocus & loc = m_structures.GetLocus(spec.GetLocusName());
            loci.push_back(&loc);
        }
        catch(const gc_missing_locus& e)
        {
            loc_match locmatchType = locMatch.GetLocMatchType();
            if(locmatchType == locmatch_DEFAULT || locmatchType == locmatch_LINKED)
                // EWFIX.BUG674 -- take out single region
            {
                size_t lineNumber = parseRef.GetParseLocus(i).GetLineNumber();
                wxString fileName = parseRef.GetFileRef().GetName();
                gcCreationInfo creationInfo = gcCreationInfo::MakeDataFileCreationInfo(lineNumber,fileName);
                try
                {
                    gcRegion & region = m_structures.GetRegion(spec.GetRegionName());
                    gcLocus & loc = m_structures.MakeLocus(region,spec.GetLocusName(),spec.GetBlessedLocus(),creationInfo);
                    gcGeneralDataType dType = parseRef.GetDataType();
                    if(dType.size()== 1)
                    {
                        loc.SetDataType(*dType.begin());
                    }
                    loci.push_back(&loc);
                }
                catch(const missing_region& r)
                {
                    gcRegion & region = m_structures.MakeRegion(spec.GetRegionName(),spec.GetBlessedRegion());
                    gcLocus & loc = m_structures.MakeLocus(region,spec.GetLocusName(),spec.GetBlessedLocus(),creationInfo);
                    gcGeneralDataType dType = parseRef.GetDataType();
                    if(dType.size() == 1)
                    {
                        loc.SetDataType(*dType.begin());
                    }
                    loci.push_back(&loc);
                }
            }
            else
            {
                throw;
            }
        }
    }

    return loci;
}

popVector
GCDataStore::GetPopsFor(const GCParse & parseRef, const GCPopMatcher & popMatch)
{
    popVector pops;

    for(size_t i=0; i < parseRef.GetPopCount(); i++)
    {
        GCPopSpec spec = popMatch.GetPopSpec(i,parseRef);
        try
        {
            gcPopulation & pop = m_structures.GetPop(spec.GetName());
            pops.push_back(&pop);
        }
        catch(const gc_missing_population& e)
        {
            if(popMatch.GetPopMatchType() == popmatch_DEFAULT)
            {
                gcPopulation & pop = m_structures.MakePop(spec.GetName(),spec.GetBlessed());
                pops.push_back(&pop);
            }
            else
            {
                throw;
            }
        }
    }

    return pops;
}

bool
GCDataStore::CanAssignParseLocus(const GCParseLocus & pLocus, const gcLocus & locus) const
{
    // EWFIX.P3 -- refactor with AssignParseLocus
    gcSpecificDataType  locusType   = locus.GetDataType();
    gcGeneralDataType   parseType   = pLocus.GetDataType();

    if(locusType != sdatatype_NONE_SET)
    {
        if(parseType.find(locusType) == parseType.end())
        {
            return false;
        }
    }
    if(locus.HasNumMarkers())
    {
        if(locus.GetNumMarkers() != pLocus.GetNumMarkers())
        {
            return false;
        }
    }
    return true;
}

void
GCDataStore::AssignParseLocus(const GCParseLocus & pLocus, gcLocus & locus)
{
    gcSpecificDataType  locusType   = locus.GetDataType();
    gcGeneralDataType   parseType   = pLocus.GetDataType();

    if(locusType != sdatatype_NONE_SET)
    {
        if(parseType.find(locusType) == parseType.end())
        {
            throw gc_locus_types_mismatch(pLocus.GetName(),locus.GetName(),ToWxString(parseType),ToWxString(locusType));
        }
    }
    else
    {
        if(parseType.size() == 1)
        {
            locus.SetDataType(*(parseType.begin()));
        }
    }

    // number of markers
    if(locus.HasNumMarkers())
    {
        if(locus.GetNumMarkers() != pLocus.GetNumMarkers())
        {
            throw gc_locus_site_count_mismatch(pLocus.GetLongName(),locus.GetLongName(),pLocus.GetNumMarkers(),locus.GetNumMarkers());
        }
    }
    else
    {
        locus.SetNumMarkers(pLocus.GetNumMarkers());
    }
}

void
GCDataStore::AssignPop(const GCParsePop & pPop, gcPopulation & pop)
{
    // EWFIX.P5 LATER -- right now all pops are mergeable
}

void
GCDataStore::SetParseChoice(const GCParse &         parseRef,
                            const GCPopMatcher &    popMatch,
                            const GCLocusMatcher &  locMatch)
{
    // remove assignments to old parse
    const GCFile & fileRef = parseRef.GetFileRef();
    if(m_structures.HasParse(fileRef))
    {
        const GCParse & oldParse = m_structures.GetParse(fileRef);
        m_structures.UnsetParse(oldParse);
        UnsetParseChoice(m_structures.GetParse(fileRef));
    }

    // check we can assign to these pops
    popVector pops = GetPopsFor(parseRef,popMatch);
    assert (pops.size() == parseRef.GetPopCount());
    for(size_t i=0; i < pops.size(); i++)
    {
        const GCParsePop & parsePop = parseRef.GetParsePop(i);
        gcPopulation & pop = *(pops[i]);
        AssignPop(parsePop,pop);
    }

    locVector locs = GetLociFor(parseRef,locMatch);
    assert (locs.size() == parseRef.GetLociCount());

    for(size_t lIndex = 0; lIndex < locs.size(); lIndex++)
    {
        const GCParseLocus & parseLocus = parseRef.GetParseLocus(lIndex);
        gcLocus & locus = *(locs[lIndex]);
        AssignParseLocus(parseLocus,locus);

        for(size_t pIndex = 0; pIndex < pops.size(); pIndex++)
        {
            const GCParseBlock & block = parseRef.GetBlock(pIndex,lIndex);
            size_t blockId = block.GetId();
            m_structures.AssignBlock(blockId,pops[pIndex]->GetId(),locs[lIndex]->GetId());
        }
    }

    // if we make it this far, update the parse in structures
    m_structures.SetParse(parseRef);

    // and document the pop and locus matchers in file info
    GetStructures().SetPopMatcher(fileRef,popMatch);
    GetStructures().SetLocusMatcher(fileRef,locMatch);

}

void
GCDataStore::UnsetParseChoice(const GCParse & parseRef)
{
    //
    constBlockVector blocks = parseRef.GetBlocks();
    for(constBlockVector::iterator i=blocks.begin(); i != blocks.end(); i++)
    {
        const GCParseBlock & block = *(*i);
        m_structures.RemoveBlockAssignment(block.GetId());
    }
    m_structures.UnsetParse(parseRef);
}

const GCParse &
GCDataStore::GetParse(const GCFile & file) const
{
    return m_structures.GetParse(file);
}

const GCParse &
GCDataStore::GetParse(const GCFile & file, size_t index) const
{
    return file.GetParse(index);
}

bool
GCDataStore::HasParse(const GCFile & file) const
{
    return m_structures.HasParse(file);
}

gcGeneralDataType
GCDataStore::GetLegalLocusTypes(size_t locusId) const
{
    gcGeneralDataType allowedTypes = gcdata::allDataTypes();
    constBlockVector blocks = GetBlocksForLocus(locusId);

    for(constBlockVector::const_iterator i = blocks.begin(); i != blocks.end(); i++)
    {
        const GCParseBlock & pb = **i;
        const GCParse & parse = pb.GetParse();
        gcGeneralDataType thisType = parse.GetDataType();
        allowedTypes.Intersect(thisType);
    }
    return allowedTypes;
}

bool
GCDataStore::FileInducesHaps(size_t fileId) const
{
    const GCFile & fileRef = GetDataFile(fileId);

    for(size_t index=0; index < fileRef.GetParseCount(); index++)
    {
        const GCParse & parseRef = fileRef.GetParse(index);
        constBlockVector blocks = parseRef.GetBlocks();
        for(constBlockVector::const_iterator biter = blocks.begin();
            biter != blocks.end(); biter++)
        {
            const GCParseBlock & pb = **biter;
            const GCParseSamples & samples = pb.GetSamples();
            for(size_t j = 0; j < samples.size(); j++)
            {
                const GCParseSample & s = *(samples[j]);
                if(s.GetSequencesPerLabel() > 1)
                {
                    return true;
                }
            }
        }
    }
    return false;
}

//____________________________________________________________________________________
