// $Id: MigrateConverter.cpp,v 1.38 2018/01/03 21:32:57 mkkuhner Exp $

/*
Copyright 2002-2005 Mary K. Kuhner, Peter Beerli, and Joseph Felsenstein

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

/* Authors: Patrick Colacurcio, Peter Beerli, Mary Kuhner, Jon Yamato 
/* Authors: Patrick Colacurcio, Peter Beerli, Mary Kuhner, Jon Yamato 
and Joseph Felsenstein */


// refactored by Mary April 30 2002

#include <cassert>
#include <iostream>
#include <sstream>
#include <cctype>
#include <cstring>
#include <stdio.h> // for sprintf in MigrateConverter::getNewName
#include <vector>
#include <algorithm>

#include "Converter_MigrateConverter.h"
#include "Converter_DataSourceException.h"
#include "Converter_LamarcDS.h"
#include "Converter_RegionDS.h"
#include "Converter_PopulationDS.h"
#include "Converter_IndividualDS.h"
#include "Converter_Sequence.h"
#include "stringx.h"
#include "random.h"

#ifdef DMALLOC_FUNC_CHECK
#include "/usr/local/include/dmalloc.h"
#endif

using namespace std;

//------------------------------------------------------------------------------------

MigrateConverter::MigrateConverter(const string& fileName, bool interleaved)
    : ConverterIf(),
      m_fileName(fileName),
      m_interleaved(interleaved),
      m_inFile(fileName.c_str(), ios::in),  // open the file
      m_markerstoregions(false) // default to not treat each marker as
      // a region
{
    // Open the file.
    if (!m_inFile)
    {
        // Cant find the infile.
        // Bail
        throw FileNotFoundError ("Sorry.  Can't find the file: " + fileName + ".  Please check your path.");
    }

    // Read the first line and parse for datatype.
    m_firstLine.erase();

    if (!getLine(m_inFile, m_firstLine))
    {
        throw FileFormatError ("Your file appears to be empty.");
    }
    istringstream firstLineStream(m_firstLine);

    m_datatype = ReadDataType(firstLineStream);

} // MigrateConverter ctor

//------------------------------------------------------------------------------------

void MigrateConverter::ProcessData()
{
    istringstream firstLineStream(m_firstLine);

    // We've already read the datatype, but need to get it out of the
    // stream, so we will read it again and throw it away.
    ReadDataType(firstLineStream);

    if (m_interleaved && m_datatype == lamarcstrings::MICROSAT)
    {
        string errormsg = "Don't know how to convert an interleaved ";
        errormsg += "Migrate/Recombine file containing microsatellite ";
        errormsg += "data";
        throw FileFormatError(errormsg);
    }
    if (m_interleaved && m_datatype == lamarcstrings::KALLELE)
    {
        string errormsg = "Don't know how to convert an interleaved ";
        errormsg += "Migrate/Recombine file containing allelic ";
        errormsg += "data";
        throw FileFormatError(errormsg);
    }

    //  Next, lets get the number of populations and the number of loci.
    skipWhiteSpace(firstLineStream);

    //  Get the number of populations.
    long numPops = GetNumPops(firstLineStream);

    //  Get the number of loci.
    long numLoci = GetNumLoci(firstLineStream);

    string delim;
    if (m_datatype == lamarcstrings::MICROSAT)
    {
        delim = ".";   // EWFIX.P3 -- default
    }
    if (m_datatype == lamarcstrings::MICROSAT || m_datatype == lamarcstrings::KALLELE)
    {
        // Get the delimiter character
        string mightBeDelim;
        if (getToken(firstLineStream,mightBeDelim))
        {
            if(mightBeDelim.length() == 1)
            {
                delim = mightBeDelim;
            }
        }
    }

    //  Process the second line, in the case of DNA or SNP data.
    vector<long> sequenceLengths;
    if (m_datatype == lamarcstrings::DNA || m_datatype == lamarcstrings::SNP)
    {
        string secondLine;
        //  Lets get the second line and put it into its own stream
        if (!getLine(m_inFile, secondLine))
        {
            throw FileFormatError ("Your file appears to be empty.");
        }

        istringstream secondLineStream(secondLine);

        //  Next, lets get the length of the sequences.
        sequenceLengths = GetSeqLengths(secondLineStream, numLoci);
    }

    //  For each population, we want to process the top line, and then process the sequence
    //  for EACH locus.

    for (int i = 0; i < numPops; i++)
    {
        //  Process the third line
        string thirdLine;
        //  Lets get the third line and put it into its own stream
        if (!getLine(m_inFile, thirdLine))
        {
            throw FileFormatError ("Your file appears to be missing a line specifing the number of sequences in a population.");
        }

        istringstream thirdLineStream(thirdLine);
        long numSequences = 0;

        //  Next, lets get number of individuals.
        numSequences = GetNumSeqs(thirdLineStream);

        // Grab the Population name if its there.
        string popName = "";
        getLine(thirdLineStream, popName);
        if (popName.empty())
            popName = string("Population") + ToString(i);

        //  Now.  Grab the sequence and individual names for each locus.
        if (m_datatype == lamarcstrings::MICROSAT || m_datatype == lamarcstrings::KALLELE)
        {
            string regionName = buildName(m_fileName,"--",3,99); // EWFIX.CONSTANT

            GetMicroSatLoci(m_inFile,
                            numSequences,
                            numLoci,
                            popName,
                            regionName,
                            m_datatype,
                            delim);
        }
        else
        {
            for (int j = 0; j < numLoci; j++)
            {
                string regionName = buildName(m_fileName,"--",3,j); // EWFIX.CONSTANT

                //  Get the sequence lengths in order.
                long sequenceLength = sequenceLengths[j];
                //  TODO  Error catching required here?

                skipWhiteSpace(m_inFile);
                if (m_interleaved)
                {
                    getInterleavedSequences(m_inFile,
                                            numSequences,
                                            sequenceLength,
                                            popName,
                                            regionName,
                                            m_datatype);
                }
                else
                {
                    getNonInterleavedSequences(m_inFile,
                                               numSequences,
                                               sequenceLength,
                                               popName,
                                               regionName,
                                               m_datatype);
                }
            }
        }
    }
}                               // MigrateConverter::ProcessData

//------------------------------------------------------------------------------------

void MigrateConverter::SetMarkersToRegions(bool val)
{
    m_markerstoregions = val;
} // SetMarkersToRegions

//------------------------------------------------------------------------------------

MigrateConverter::~MigrateConverter ()
{
}                               // MigrateConverter::~MigrateConverter

void MigrateConverter::getInterleavedSequences(ifstream& inFile,
                                               const long numSequences,
                                               const long sequenceLength,
                                               const string& popName,
                                               const string& regionName,
                                               const string& datatype)
{
    vector<string> names;
    vector<string> sequences;

    for ( int i = 0; i < numSequences; i++)
    {
        string lineString;
        string sequenceStr;

        // get the first line
        if (getLine (inFile, lineString))
        {
            istringstream indNameStream(lineString);
            string indName;

            // Get the first 10 characters.
            if (!getNextNChars(indNameStream, indName, 10))
                throw FileFormatError ("Your file is not in the correct Migrate format.  Please check your file and retry.");

            //  Get the rest of the line.  (don't know whether there will be spaces, so get as tokens
            while (getToken(indNameStream, sequenceStr))
            {};

            //  After getting the name and the sequence
            //  1) reset the seqName string
            //  2) stuff both the name and the sequence into the maps
            //  3) skip whitespace so the next line is ready to go

            names.push_back(indName);
            sequences.push_back(sequenceStr);
            skipWhiteSpace(inFile);
        }
    }

    //  Now, get the rest of the interleaved sequences;
    bool continuePullingSequence = true;

    //  If there are no sequences, don't pull.
    if (numSequences == 0)
        continuePullingSequence = false;

    int i = 0;

    while (continuePullingSequence)
    {
        string lineString;
        if (getLine (inFile, lineString))
        {
            istringstream indNameStream(lineString);
            string sequenceStr;

            //  Get the rest of the line.  (don't know whether there will be spaces, so get as tokens.
            while (getToken(indNameStream, sequenceStr)) {};

            sequences[i%numSequences] = sequences[i%numSequences] + sequenceStr;

            skipWhiteSpace(inFile);
        }
        else
        {
            throw FileFormatError ("Your file is not in the correct Migrate format.  It appears that one of the sequences is too short.  The total sequence is: " + (sequences[i%numSequences]));
        }

        if( (i%numSequences) == (numSequences - 1))
        {
            //          cout << names[i%numSequences] << " length " << sequences[i%numSequences].length() << ":"
            //               << sequences[i%numSequences] << endl;

            if( (long)(sequences[i%numSequences].length()) >= sequenceLength)
            {
                continuePullingSequence = false;
            }
        }
        i++;
    }

    //  Now, Put together the LamarcDS.
    //  Put together the individuals and their sequences.
    PopulationDS pop (popName);

    //  Add a comment for this population.
    string comment = "Population origin file for " + regionName + "/";
    comment += popName + ": " + m_fileName;
    pop.setComment(comment);

    long seq, nseqs = sequences.size();
    for(seq = 0; seq < nseqs; ++seq)
    {
        IndividualDS individual(names[seq],sequences[seq],datatype);
        pop.addIndividual(individual);
    }

    //  Add the region.
    ModelDS model;
    RegionDS region (regionName, model, pop);

    m_lamarc.addRegion(region);
}  //getInterleavedSequences

void MigrateConverter::getNonInterleavedSequences(ifstream& inFile,
                                                  const long numSequences,
                                                  const long sequenceLength,
                                                  const string& popName,
                                                  const string& regionName,
                                                  const string& datatype)
{
    vector<string> names;
    vector<string> sequences;

    //  If its not interleaved, its a bit easier.
    PopulationDS pop (popName);
    skipWhiteSpace(inFile);

    for ( int i = 0; i < numSequences; i++)
    {
        string indName;
        string sequenceStr;

        // Get the first 10 characters.
        if (!getNextNChars(inFile, indName, 10))
            throw FileFormatError ("Your file is not in the correct Migrate format.  \nPlease check your file and retry.");

        //  Get characters until we've gone through the sequence length.
        if (!getNextNNonWhiteSpace(inFile, sequenceStr, sequenceLength))
        {
            throw FileFormatError ("Your file is not in the correct Migrate format. \nProbably one of the sequences is too short.");
        }

        // Got the name, got the sequence, but can't stuff them into
        // individuals yet if haplotyping is not known.
        sequences.push_back(sequenceStr);
        names.push_back(indName);

        //  Clean up for the next sequence
        skipWhiteSpace(inFile);
    }

    long seq, nseqs = sequences.size();
    for(seq = 0; seq < nseqs; ++seq)
    {
        IndividualDS individual(names[seq],sequences[seq],datatype);
        pop.addIndividual(individual);
    }

    //  Now, Put together the LamarcDS
    //  Add the region
    ModelDS model;
    RegionDS region (regionName, model, pop);

    m_lamarc.addRegion(region);
} //getNonInterleavedSequences

//------------------------------------------------------------------------------------

void MigrateConverter::GetMicroSatLoci(ifstream& inFile,
                                       const long numSequences,
                                       const long numMarkers,
                                       const string& popName,
                                       const string& regionName,
                                       const string& datatype,
                                       const string& delimiter)
{
    vector<string> names;
    vector< pair<vector<string>,vector<string> > > microsats;

    PopulationDS pop (popName);
    skipWhiteSpace(inFile);

    for ( int i = 0; i < numSequences; i++)
    {
        // Read an individual.
        string lineString;
        if (getLine (inFile, lineString))
        {
            istringstream linestream(lineString);
            // Get the first 10 characters.
            string indname;
            if (!getNextNChars(linestream,indname,10L))
            {
                string errormsg = "Not a legal Migrate/Recombine file. ";
                errormsg += "Error in trying to add individual ";
                errormsg += ToString(i) + " of population " + popName;
                throw FileFormatError(errormsg);
            }

            // Read the microsats themselves.
            vector<string> micro1;
            vector<string> micro2;
            long marker;
            for(marker = 0; marker < numMarkers; ++marker)
            {
                string haplotypes;
                if (!getToken(linestream,haplotypes))
                {
                    string errormsg = "Unable to find any haplotypes ";
                    errormsg += "of marker " + indexToKey(marker) + " ";
                    errormsg += "of individual " + indname + " of ";
                    errormsg += "population " + popName;
                    throw FileFormatError(errormsg);
                }

                // Make sure the correct delimiter is present.
                unsigned long delimiterpos = haplotypes.find(delimiter);
                if (delimiterpos == haplotypes.length())
                {
                    string errormsg = "Unable to find the delimiter ";
                    errormsg += delimiter + " in marker ";
                    errormsg += ToString(marker) + " of individual ";
                    errormsg += indname + " of population " + popName;
                    throw FileFormatError(errormsg);
                }

                string number1(haplotypes.substr(0,delimiterpos));
                if (number1.empty())
                {
                    string errormsg = "Unable to find 1st haplotype ";
                    errormsg += "of marker " + ToString(marker) + " ";
                    errormsg += "of individual " + indname + " of ";
                    errormsg += "population " + popName;
                    throw FileFormatError(errormsg);
                }
                micro1.push_back(number1);

                string number2(haplotypes.substr(delimiterpos+1,
                                                 haplotypes.length()));
                if (number2.empty())
                {
                    string errormsg = "Unable to find 2nd haplotype ";
                    errormsg += "of marker " + ToString(marker) + " ";
                    errormsg += "of individual " + indname + " of ";
                    errormsg += "population " + popName;
                    throw FileFormatError(errormsg);
                }
                micro2.push_back(number2);
            }

            StripLeadingSpaces(indname);
            StripTrailingSpaces(indname);
            names.push_back(indname);
            microsats.push_back(make_pair(micro1,micro2));
        }
        else
        {
            string errormsg = "Expected to find " + ToString(numSequences);
            errormsg += " individuals, but only found " + ToString(i-1);
            errormsg += " in population " + popName;
            throw FileFormatError(errormsg);
        }
    }

    if (m_markerstoregions)
    {
        long reg, nregs = microsats[0].first.size();
        for(reg = 0; reg < nregs; ++reg)
        {
            pop.EraseIndividuals();

            long ind, nind = microsats.size();
            for(ind = 0; ind < nind; ++ind)
            {
                Sequence seq1(microsats[ind].first[reg],datatype,
                              names[ind]+string("-1"));
                Sequence seq2(microsats[ind].second[reg],datatype,
                              names[ind]+string("-2"));
                IndividualDS individual(names[ind],seq1);
                individual.AddSequence(seq2);
                pop.addIndividual(individual);
            }
            //  Now, put together the LamarcDS.
            //  Add the region.
            ModelDS model;
            string rname = buildName(m_fileName,"--",3,reg); // EWFIX.CONSTANT
            RegionDS region (rname, model, pop);

            m_lamarc.addRegion(region);
        }
    }
    else
    {
        long ind, nind = microsats.size();
        assert(nind == numSequences);
        for(ind = 0; ind < nind; ++ind)
        {
            Sequence seq1(microsats[ind].first,datatype,
                          names[ind]+string("-1"));
            Sequence seq2(microsats[ind].second,datatype,
                          names[ind]+string("-2"));
            IndividualDS individual(names[ind],seq1);
            individual.AddSequence(seq2);
            pop.addIndividual(individual);
        }

        //  Now, put together the LamarcDS.
        //  Add the region.
        ModelDS model;
        RegionDS region (regionName, model, pop);

        m_lamarc.addRegion(region);
    }
} // GetMicroSatLoci

//------------------------------------------------------------------------------------

string
MigrateConverter::getNewName(Random &random) const
{
    int char1 = abs(random.Long() % 26) + 65;
    int char2 = abs(random.Long() % 26) + 65;
    int char3 = abs(random.Long() % 26) + 65;
    int char4 = abs(random.Long() % 26) + 65;
    char name[6];

    sprintf(name, "%c%c%c%c", char1, char2, char3, char4);

    string newName(name);

    return newName;
}

//  Okay.  Here is the deal.  Send in a lamarc, and it will merge this converter's lamarc into yours.
//  If you want a lamarc of your own, send in an empty one.  I can't return a lamarc or do assignment
//  on a lamarc because I'm having really, really weird template issues.
void
MigrateConverter::addConvertedLamarcDS (LamarcDS& lamarc)
{
    lamarc.mergeTo(m_lamarc);
}

long MigrateConverter::GetNumPops(istringstream& linestream) const
{
    //  Get the number of populations
    //  (relies on short-circuit evaluation)
    string someString;
    long value;
    if (getNumber(linestream, someString) && IsInteger(someString))
    {
        if (FromString(someString,value)) return value;
        else return 0L;
    }
    else
    {
        // Bail.
        throw FileFormatError ("Your file is not in the correct Migrate format.\nThe first number must be an integer signaling the number of Populations in the file.");
    }

    assert(false); // can't get here
    return 0L;
} // GetNumPops

long MigrateConverter::GetNumLoci(istringstream& linestream) const
{

    //  Get the number of loci.
    //  (Relies on short-circuit evaluation.)
    string someString;
    long value;
    if (getNumber(linestream, someString) && IsInteger(someString))
    {
        if (FromString(someString,value)) return value;
        else return 0L;
    }
    else
    {
        // Bail.
        throw FileFormatError ("Your file is not in the correct Migrate format.\nThe second number must be an integer signaling the number of Loci in the file.");
    }

    assert(false); // can't get here
    return 0L;
} // GetNumLoci

vector<long> MigrateConverter::GetSeqLengths(istringstream& linestream, long numLoci) const
{
    //  Next, lets get the length of the sequences.
    //  (Relies on short-circuit evaluation.)
    string someString;
    vector<long> sequenceLengths;
    long length;

    for (int i = 0; i < numLoci; i++)
    {
        if (getNumber(linestream, someString) && IsInteger(someString))
        {
            if (!FromString(someString,length)) length = 0;
            sequenceLengths.push_back(length);
            someString = "";
        }
        else
        {
            // Bail.
            string errormsg = "Your file is not in the correct Migrate format.\n";
            errormsg += "The second line must contain one integer per locus specifying \n";
            errormsg += "the length of the sequences in the file.";
            throw FileFormatError (errormsg);
        }
    }

    return sequenceLengths;

} // GetSeqLengths

string MigrateConverter::ReadDataType(istringstream& linestream) const
{
    if (isFirstChar(linestream,string("sSnNmMeE")))
    {
        string datatype;
        getWord(linestream, datatype);
        if (strpbrk(datatype.c_str(),string("sS").c_str()))
            return string("DNA");
        if (strpbrk(datatype.c_str(),string("nN").c_str()))
            return string("SNP");
        if (strpbrk(datatype.c_str(),string("mM").c_str()))
            return string("MICROSAT");
        if (strpbrk(datatype.c_str(),string("eE").c_str()))
            return string("KALLELE");
        return string("");   // cannot be reached
    }
    else
    {
        return string("");
    }

} // ReadDataType

long MigrateConverter::GetNumSeqs(istringstream& linestream) const
{
    //  Next, lets get number of individuals.
    // Relies on short-circuit evaluation.
    string someString;
    long value;

    if (getNumber(linestream, someString) && IsInteger(someString))
    {
        if (!FromString(someString,value)) value = 0;
        return value;
    }
    else
    {
        // Bail.
        string errormsg = "Your file is not in the correct Migrate format.  At the \n";
        errormsg += "beginning of each population, there must be a line specifying the number\n";
        errormsg += "of sequences in that population.";
        throw FileFormatError(errormsg);
    }
    assert(false); // can't get here
    return 0L;

} // GetNumSeqs

//____________________________________________________________________________________
