/*------------------------------------------------------------------------------*
 * File Name:				 													*
 * Creation: 																	*
 * Purpose: OriginC Source C file												*
 * Copyright (c) ABCD Corp.	2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007		*
 * All Rights Reserved															*
 * 																				*
 * Modification Log:															*
 * EJP 07-07-2003 v7.0619 QA70-4783 POST_IMPORT_EXECUTE							*
 * EJP 07-28-2003 v7.0637 QA70-4911 IMPROVE_HDR_VAR_SCAN						*
 * EJP 07-29-2003 v7.0637 QA70-4898 PREPARE_WKS_FOR_IMPORT_NEED_USE_CORRECT_NUM_COLUMNS
 * EJP 08-07-2003 v7.0648 QA70-4934 FIX_COL_DESIGNATIONS_AND_FORMATS_SET		*
 * EJP 08-08-2003 v7.0650 QA70-4073.53 USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP
 * EJP 08-12-2003 v7.0656 QA70-4808 IMPORT_INTO_LOCKED_COL_WARNING				*
 * EJP 08-12-2003 v7.0657 HIDE_DATA_WINDOW_ON_IMPORT_INTO_GRAPH					*
 * EJP 08-29-2003 v7.5680 IMPROVE_IMPORT_SPEED_OF_MANY_COLS						*
 * EJP 09-15-2003 v7.5695 QA70-5135.19 AVOID_UNWANTED_REPEATING_OF_LAST_COL_DESIG_AND_FORMAT
 * EJP 09-22-2003 v7.5701 QA70-4818 SAVE_FILTER_IN_WKS							*
 * EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR					*
 * EJP 09-26-2003 v7.5707 QA70-5239 IMPORT_INTO_TEXTNUMERIC_COLUMNS				*
 * EJP 11-12-2003 v7.5753 QA70-5211.11 FIX_SETTING_OF_DESIG_AND_FORMAT			*
 * EJP 11-24-2003 v7.5769 QA70-5417.18 FIX_DESIG_AND_FMT						*
 *------------------------------------------------------------------------------*/
 
////////////////////////////////////////////////////////////////////////////////////
// you can include just this typical header file for most Origin built-in functions and classes
// and it takes a reasonable amount of time to compile, 
#include <origin.h>

#include "FileImport.h"
#include "App_Utils.h"
#include "binimp.h"
#include "wks_utils.h"
#include "Filter_Utils.h"

bool iwGetFilter(LPCSTR lpcszDataFile, string &strFilterFile, TreeNode &trFilter);

#define PAGEINFO_IMPORTHEADER "ImportHeader"

////////////////////////////////////////////////////////////////////////////////////
static string s_strFilter; // filter file name
static string s_strGraphPage; // name of target graph page

/// EJP 08-08-2003 v7.0650 QA70-4073.53 USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP
static Tree s_trFilter; // filter settings
/// end USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP

////////////////////////////////////////////////////////////////////////////////////
// start your functions here

BOOL HandleDataFile()
{
	string strWinName;
	int	nLayer;
	int nFile;
	int	nNumFiles;
	string strFileName;
	Project.GetDragDropInfo(strWinName, nLayer, nFile, nNumFiles, strFileName);

	if( 0 == nFile )
	{
		s_strFilter.Empty();
		s_strGraphPage.Empty();
		
		/// EJP 08-08-2003 v7.0650 QA70-4073.53 USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP
		s_trFilter.Reset();
		/// end USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP
	}
	
	/// EJP 08-08-2003 v7.0650 QA70-4073.53 USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP
	/*
	if( s_strFilter.IsEmpty() )
	{
		if( 1 != fuGetFilterFile(s_strFilter, strFileName) )
			return false;
	}

	Page pgData(strWinName);

	if( ImportFile(pgData, s_strFilter, strFileName, nFile) )
		return false;

	Tree trFilter;
	if( !trFilter.Load(s_strFilter) )
		return false;

	GraphPage grpg;
	grpg = GetTargetGraphPage(trFilter, strWinName, nLayer);
	if( grpg )
	{
		int nPlotID = fuGetPlotID(trFilter);
		GraphData(grpg, nLayer, nPlotID, pgData);
	}
	*/
	if( !fuIsApplicable(s_trFilter, strFileName) && !fuIsApplicable(s_strFilter, strFileName) )
	{
		/// EJP 09-22-2003 v7.5701 QA70-4818 SAVE_FILTER_IN_WKS
		Page pgTarget(strWinName);
		int iFilterType;
		bool bFilterInPage = false;
		if( fuIsOneFilterInPage(pgTarget, strFileName, iFilterType) )
			bFilterInPage = fuLoadFilterFromPage(s_trFilter, pgTarget, iFilterType);
		if( !bFilterInPage )
		{
		/// end SAVE_FILTER_IN_WKS
			if( 1 != fuGetFilterFile(s_strFilter, strFileName) )
			{
				if( !iwGetFilter(strFileName, s_strFilter, s_trFilter) )
					return FALSE;
			}
		/// EJP 09-22-2003 v7.5701 QA70-4818 SAVE_FILTER_IN_WKS
		}
		/// end SAVE_FILTER_IN_WKS
	}
	
	int iErr; /// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR

	Page pgData(strWinName);
	if( s_strFilter.IsEmpty() )
	{
		/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
		///if( ImportFile(pgData, s_trFilter, strFileName, nFile) )
		///	return false;
		iErr = ImportFile(pgData, s_trFilter, strFileName, nFile);
		/// end NOTIFY_USER_OF_IMPORT_ERROR
	}
	else
	{
		/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
		///if( ImportFile(pgData, s_strFilter, strFileName, nFile) )
		///	return false;
		///if( !s_trFilter.Load(s_strFilter) )
		///	return false;
		iErr = ImportFile(pgData, s_strFilter, strFileName, nFile);
		if( IMPERR_NONE == iErr )
		{
			if( !s_trFilter.Load(s_strFilter) )
				iErr = IMPERR_LOAD_FILTER;
		}
		/// end NOTIFY_USER_OF_IMPORT_ERROR
	}
	
	/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
	if( IMPERR_NONE != iErr )
	{
		ImportErrorMsgBox(iErr, s_strFilter);
		return false;
	}
	/// end NOTIFY_USER_OF_IMPORT_ERROR
	
	GraphPage grpg;
	grpg = GetTargetGraphPage(s_trFilter, strWinName, nLayer);
	if( grpg )
	{
		int nPlotID = fuGetPlotID(s_trFilter);
		GraphData(grpg, nLayer, nPlotID, pgData);
	}
	/// end USE_SAME_FILTER_FOR_MULTI_FILES_ON_FILEOPEN_AND_DRAGDROP

	return true;
}

int ImportFile(Page &pgTarget, LPCSTR lpcszFilter, LPCSTR lpcszFile, int nFile)
{
	if( NULL == lpcszFilter )
		return IMPERR_NO_FILTER; // no filter
	
	Tree trFilter;
	if( !trFilter.Load(lpcszFilter) )
		return IMPERR_LOAD_FILTER; // load filter error
	
	/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
	///if( ImportFile(pgTarget, trFilter, lpcszFile, nFile) )
	///	return 1; // import error
	int iErr = ImportFile(pgTarget, trFilter, lpcszFile, nFile);
	if( IMPERR_NONE != iErr )
		return iErr;
	/// end NOTIFY_USER_OF_IMPORT_ERROR

	if( pgTarget )
		pgTarget.Info.System.Import.Filter$ = lpcszFilter;

	return IMPERR_NONE; // success
}

int ImportFile(Page &pgTarget, TreeNode &trFilter, LPCSTR lpcszFile, int nFile)
{
	int iFilterType = fuGetFilterType(trFilter);
	
	// Get the target data page.
	pgTarget = GetTargetDataPage(trFilter, pgTarget, nFile);
	if( !pgTarget && FILTER_TYPE_USERDEFINED != iFilterType )
		return IMPERR_NO_TARGET_PAGE; // no target page

	// If target page is a wks then setup the columns.
	string strColDesig, strColFormat;
	if( pgTarget && EXIST_WKS == pgTarget.GetType() )
	{
		/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
		///	/// EJP 08-12-2003 v7.0656 QA70-4808 IMPORT_INTO_LOCKED_COL_WARNING
		///	///PrepareWksColsForImport(pgTarget, trFilter, lpcszFile);
		///	if( PrepareWksColsForImport(pgTarget, trFilter, lpcszFile) )
		///		return 1;
		///	/// end IMPORT_INTO_LOCKED_COL_WARNING
		int iErr = PrepareWksColsForImport(pgTarget, trFilter, lpcszFile);
		if( IMPERR_NONE != iErr )
			return iErr;
		/// end NOTIFY_USER_OF_IMPORT_ERROR
		Worksheet wks(pgTarget.GetName());
		if( wks )
		{
			strColDesig = wks.GetColDesignations();
			strColFormat = wks.GetColFormats();
		}

		/// EJP 11-24-2003 v7.5769 QA70-5417.18 FIX_DESIG_AND_FMT
		///	/// EJP 09-26-2003 v7.5707 QA70-5239 IMPORT_INTO_TEXTNUMERIC_COLUMNS
		///	// Importing into Text columns (and maybe others) will fail.
		///	// Instead we import into Text&Numeric and set user specified format afterwards.
		///	wks.SetColFormats("9", true); // temporary set all cols to Text&Numeric
		///	/// end IMPORT_INTO_TEXTNUMERIC_COLUMNS
		/// end FIX_DESIG_AND_FMT
	}

	// Get pointer to import function.
	PFNIMPORTFUNC pfn = fuGetImportFunctionPtr(trFilter);
	if( !pfn )
		return IMPERR_GET_IMPORT_FUNC; // no import function

	// Call import function.
	if( pfn(pgTarget, trFilter, lpcszFile, nFile) )
		return IMPERR_IMPORT_FUNC_ERR; // import function error

	if( pgTarget && EXIST_WKS == pgTarget.GetType() && FILTER_TYPE_USERDEFINED != iFilterType )
	{
		Worksheet wks(pgTarget.GetName());
		if( wks )
		{
			wks.SetColDesignations(strColDesig, false);
			wks.SetColFormats(strColFormat, false);
		}
	}
	
	// Check if not ASCII type because the intenal ASCII import code
	// will handle renaming the wks.
	if( FILTER_TYPE_ASCII != iFilterType && fuIsRenameWks(trFilter) )
	{
		string strNewName = GetFileName(lpcszFile, TRUE);
		if( !strNewName.IsEmpty() )
			pgTarget.Rename(strNewName);
	}
	
	if( pgTarget && fuIsFileNameToWksLabel(trFilter) )
	{
		pgTarget.Label = GetFileName(lpcszFile);
		pgTarget.TitleShow = WIN_TITLE_SHOW_BOTH;
	}
	
	// If data file has header parameters then add them to page info.
	if( pgTarget && fuGetHeaderParamCount(trFilter) )
		HeaderVariablesToPageInfo(pgTarget, trFilter, lpcszFile);

	/// EJP 07-10-2003 v7.0622 QA70-4745 SET_PAGE_IMPORT_INFO_ON_123_ASC_IMPORT
	/*
	// Set page import info.
	/// EJP 06-30-2003 v7.0613 QA70-4745 ADD_SYSTEM_IMPORT_FILEDATE
	double dJulianDate = 0.0;
	WIN32_FILE_ATTRIBUTE_DATA fad;
	if( GetFileAttributesEx(lpcszFile, 0, &fad) )
	{
		SYSTEMTIME st;
		if( FileTimeToSystemTime(&fad.ftLastWriteTime, &st) )
			SystemTimeToJulianDate(&dJulianDate, &st);
	}
	pgTarget.Info.System.Import.FileDate = dJulianDate;
	/// end ADD_SYSTEM_IMPORT_FILEDATE
	pgTarget.Info.System.Import.FileName$ = GetFileName(lpcszFile);
	pgTarget.Info.System.Import.FilePath$ = lpcszFile;
	pgTarget.Info.System.Import.Filter$ = ""; // filter file unknown
	pgTarget.Info.System.Import.FileType = trFilter.Type.nVal;
	*/
	set_page_import_info(pgTarget, lpcszFile, iFilterType);
	/// end SET_PAGE_IMPORT_INFO_ON_123_ASC_IMPORT

	// Save any requested header lines.
	if( FILTER_TYPE_ASCII == iFilterType )
	{
		int iFirstLine, iNumLines;
		fuGetHdrSave(trFilter, iFirstLine, iNumLines);
		if( iNumLines )
		{
			string strHdrLines;
			if( 0 == LoadTextFile(strHdrLines, lpcszFile, iNumLines, iFirstLine) )
			{
				vector<byte> vb;
				if( strHdrLines.GetBytes(vb) )
					pgTarget.SetMemory(PAGEINFO_IMPORTHEADER, vb);
			}
		}
	}

	/// EJP 07-07-2003 v7.0619 QA70-4783 POST_IMPORT_EXECUTE
	if( fuGetPostImportScript(trFilter, strColDesig) && !strColDesig.IsEmpty() )
	{
		if( pgTarget )
		{
			Layer lay = pgTarget.Layers();
			if( lay )
				lay.LT_execute(strColDesig);
		}
	}
	/// end POST_IMPORT_EXECUTE
	
	return IMPERR_NONE; // success
}

//--------------------------------------------------------------------------
// GetTargetDataPage
//
// Determine the target page for the data by using the filter and window
// name and layer (if drag-drop on a window).
// This function may return the page specified in the window name argument
// but that is not guranteed.  It depends on the filter settings.
//--------------------------------------------------------------------------
static Page GetTargetDataPage(TreeNode &trFilter, Page &pgTarget, int nFile)
{
	if( pgTarget && fuIsTargetPageType(trFilter, pgTarget.GetType()) )
	{
		if( 0 == nFile || ASCIMP_MODE_REPLACE_DATA != fuGetImportMode(trFilter) || IsPageEmpty(pgTarget) )
			return pgTarget; // use current page
	}

	/// EJP 08-12-2003 v7.0657 HIDE_DATA_WINDOW_ON_IMPORT_INTO_GRAPH
	///return fuCreateTargetPage(trFilter); // create a new page
	int iOption = CREATE_VISIBLE;
	if( pgTarget && EXIST_PLOT == pgTarget.GetType() )
		iOption = CREATE_HIDDEN;
	return fuCreateTargetPage(trFilter, iOption); // create a new page
	/// end HIDE_DATA_WINDOW_ON_IMPORT_INTO_GRAPH
}

static BOOL IsPageEmpty(Page &pg)
{
	string str = pg.GetName();
	if( EXIST_WKS == pg.GetType() )
	{
		WorksheetPage wkspg(str);
		if( wkspg )
		{
			Worksheet wks(str);
			if( wks )
			{
				int r1, r2;
				wks.GetRange(r1, r2, 0, -1, false);
				if( r2 == -1 ) // if wks is empty
					return TRUE;
			}
		}
	}
	else // assume matrix
	{
		MatrixPage matpg(str);
		if( matpg )
		{
/// EJP: how to check if matrix is empty?
///			MatrixLayer ml(str);
///			if( ml && !ml.HasData() )
			return TRUE;
		}
	}
	return FALSE;
}

//--------------------------------------------------------------------------
// GetTargetGraphPage
//
// Determine the target graph page for the data to be plotted in.
// This is determined using the filter and, if drag-drop, the window
// name and layer.
//--------------------------------------------------------------------------
static GraphPage GetTargetGraphPage(TreeNode &trFilter, LPCSTR lpcszWinName, int iWinLayer)
{
	string strWinName;
	if( lpcszWinName )
		strWinName = lpcszWinName;

	GraphPage pageGraph(strWinName);
	if( pageGraph ) // if DragDrop on Graph window
	{
		if( FILTER_DDGRAPH_OPENONLY == fuGetDragDropGraph(trFilter) )
			pageGraph.Detach(); // do not plot the data
	}
	else if( strWinName.IsEmpty() ) // if File Open or DragDrop on Workspace
	{
		int iDragDrop = fuGetDragDropWorkspace(trFilter);

		if( iDragDrop == FILTER_DDWORKSPACE_PLOT_1LAYER ||
			iDragDrop == FILTER_DDWORKSPACE_PLOT_MULTILAYER )
		{
			GraphPage pgTemp(s_strGraphPage);
			pageGraph = pgTemp;
		}

		if( !pageGraph && iDragDrop != FILTER_DDWORKSPACE_OPENONLY )
		{
			string str = fuGetPlotTemplate(trFilter);
			if( str.IsEmpty() )
				pageGraph.Create(); // default template
			else
				pageGraph.Create(str);
			
			if( pageGraph )
				s_strGraphPage = pageGraph.GetName();	
		}
	}

	return pageGraph;
}

//--------------------------------------------------------------------------
// GraphData
//
// Determine the target graph page for the data to be plotted in.
// This is determined using the filter and, if drag-drop, the window
// name and layer.
//
// Return:
// zero for success, non-zero for error
//--------------------------------------------------------------------------
static int GraphData(GraphPage &grpgTarget, int iLayer, int iPlotID, Page &pageData)
{
	GraphLayer grLayer = grpgTarget.Layers(iLayer);

	switch( pageData.GetType() )
	{
	case EXIST_WKS:
		Worksheet wks = pageData.Layers();
		grLayer.AddPlot(wks, iPlotID);
		break;
	case EXIST_MATRIX:
/// EJP 03-07-2003: Currently we can not get Matrix.  CP will fix.
///		Matrix mat = pageData.Layers();
///		grLayer.AddPlot(mat, iPlotID);
		break;
	default:
		return 1; // unsupported page type
	}
		
	return 0; // success
}

//--------------------------------------------------------------------------
// PrepareWksColsForImport
//
// Call this function to setup the columns for a named wks based on the
// settings in a filter.
//
// This function still does not consider partial import nor import mode
// settings.
//#define ASCIMP_MODE_REPLACE_DATA		0
//#define ASCIMP_MODE_APPEND_COLS			1
//#define ASCIMP_MODE_APPEND_ROWS			2
//--------------------------------------------------------------------------
int PrepareWksColsForImport(Page &pgTarget, TreeNode &trFilter, LPCSTR lpcszFile)
{
	int n, nCols = 0;
	string str;

	Worksheet wks(pgTarget.GetName());
	if( !wks )
		return IMPERR_PREPARE_WKS; // error, no worksheet
	
	///////////////////////////////////////////////////////
	// The following (switch on filter type) will set nCols
	// to the number of columns in the data file.
	// This number may not always be correct, but should be
	// sufficient for setting up the wks columns.
	//
	// It may be easier/safer to use the ColDesignations or
	// ColFormats setting in the filter.  I will look into
	// this later.
	///////////////////////////////////////////////////////
	switch( trFilter.Type.nVal )
	{
	case FILTER_TYPE_ASCII:
		ASCIMP ascimp;
		fuGetASCIMP(trFilter, ascimp);
		if( ascimp.iDelimited )
		{
			/// EJP 07-29-2003 v7.0637 QA70-4898 PREPARE_WKS_FOR_IMPORT_NEED_USE_CORRECT_NUM_COLUMNS
			// The wizard has been improved as far as scanning number of columns.
			// The user also now has the ability to set the number of columns.
			// Here we should assume the filter contains the correct number of columns.
			///	// This scans the file and tries to figure out
			///	// the delimiter and then counts the columns.
			///	// While the filter already contains the delimiter
			///	// there is no way to pass that to the scanning
			///	// function.
			///	ASCIMP ascimpTemp;
			///	AscImpReadFileStruct(lpcszFile, &ascimpTemp);
			///	nCols = ascimpTemp.iNumColumns;
			nCols = ascimp.iNumColumns;
			/// end PREPARE_WKS_FOR_IMPORT_NEED_USE_CORRECT_NUM_COLUMNS
		}
		else // fixed width, count tokens
		{
			// This counts the number of column widths
			// specified by the user.  This may be less than
			// the actual columns in the file since the last
			// width value is repeated until the end of line
			// is reached
			str = ascimp.szFixedWidth;
			nCols = str.GetNumTokens(',');
		}
		break;
	case FILTER_TYPE_BINARY:
		BINIMP binimp;
		fuGetBINIMP(trFilter, binimp);
		for( n = 0; n < binimp.vParamCount.GetSize(); n++ )
			nCols += binimp.vParamCount[n];
		if( nCols != binimp.iNumColumns )
		{
			binimp.iNumColumns = nCols;
			printf("BINIMP.iNumColumns updated\n");
		}
		break;
	case FILTER_TYPE_USERDEFINED:
		return IMPERR_NONE; // filter type not supported
	default:
		return IMPERR_FILTER_TYPE; // error, unknown filter type
	}

	///////////////////////////////////////////////////
	// The following will adjust nCols based on the
	// partial import settings.
	///////////////////////////////////////////////////
	int nFirstSourceCol = 0;
	if( trFilter.Common.Partial.nVal )
	{
		nFirstSourceCol = trFilter.Common.PartialC1.nVal;
		
		if( nFirstSourceCol > trFilter.Common.PartialC2.nVal )
			nCols -= nFirstSourceCol;
		else
			nCols = trFilter.Common.PartialC2.nVal - nFirstSourceCol + 1;
	}
	
	///////////////////////////////////////////////////
	// The following (switch on import mode) will
	// adjust nCols based on the import mode.
	///////////////////////////////////////////////////
	int nFirstTargetCol = 0;
	int nNumExistingCols = wks.GetNumCols();
	int nNewCols = 0;
	switch( fuGetImportMode(trFilter) )
	{
	case ASCIMP_MODE_REPLACE_DATA:
	case ASCIMP_MODE_APPEND_ROWS:
		if( nCols > nNumExistingCols )
			nNewCols = nCols - nNumExistingCols;
		break;
	case ASCIMP_MODE_APPEND_COLS:
		nFirstTargetCol = FindEmptyColumn(wks);
		if( nFirstTargetCol >= 0 )
		{
			nNewCols = nCols - (nNumExistingCols - nFirstTargetCol);
		}
		else // no empty columns
		{
			nFirstTargetCol = nNumExistingCols;
			nNewCols = nCols;
		}
		break;
	}
	/// EJP 08-29-2003 v7.5680 IMPROVE_IMPORT_SPEED_OF_MANY_COLS
	///for( n = nNumExistingCols; n < nCols; n++ )
	///wks.AddCol();
	if( nNewCols > 0 )
	{
		str.Format("work -a %d", nNewCols);
		pgTarget.LT_execute(str);
	}
	/// end IMPROVE_IMPORT_SPEED_OF_MANY_COLS

	///////////////////////////////////////////////////////
	// The follwoing will construct the designation and
	// format strings.
	///////////////////////////////////////////////////////
	bool bRepetitive = fuGetRepetitive(trFilter);

	// Start with the designations and formats in the filter.
	string strColDesig = fuGetDesignations(trFilter);
	string strColFormat = fuGetFormats(trFilter);

	// If a partial import then remove the skipped columns	
	if( nFirstSourceCol )
	{
		strColDesig.Delete(0, nFirstSourceCol);
		strColFormat.Delete(0, nFirstSourceCol);
	}
	
	/// EJP 08-07-2003 v7.0648 QA70-4934 FIX_COL_DESIGNATIONS_AND_FORMATS_SET
	/*
	if( nNumExistingCols ) // if not replacing data
	{
		if( ASCIMP_MODE_APPEND_ROWS == fuGetImportMode(trFilter) )
		{
			strColDesig.Delete(0, nNumExistingCols);
			strColFormat.Delete(0, nNumExistingCols);
		}

		string strExistColDesig = wks.GetColDesignations();
		if( strExistColDesig.GetLength() > nNumExistingCols )
			strExistColDesig.Delete(nNumExistingCols, strExistColDesig.GetLength() - nNumExistingCols);
			
		string strExistColFormat = wks.GetColFormats();
		if( strExistColFormat.GetLength() > nNumExistingCols )
			strExistColFormat.Delete(nNumExistingCols, strExistColFormat.GetLength() - nNumExistingCols);
		
		strColDesig.Insert(0, strExistColDesig);
		strColFormat.Insert(0, strExistColFormat);
	}
	*/
	if( nFirstTargetCol ) // if not importing into the first column
	{
		string strExistColDesig = wks.GetColDesignations();
		strExistColDesig.Delete(nFirstTargetCol, strExistColDesig.GetLength() - nFirstTargetCol);
		strColDesig.Insert(0, strExistColDesig);
		
		string strExistColFormat = wks.GetColFormats();
		strExistColFormat.Delete(nFirstTargetCol, strExistColFormat.GetLength() - nFirstTargetCol);
		strColFormat.Insert(0, strExistColFormat);
	}
	/// end FIX_COL_DESIGNATIONS_AND_FORMATS_SET

	/// EJP 08-12-2003 v7.0656 QA70-4808 IMPORT_INTO_LOCKED_COL_WARNING
	if( wks.IsWriteProtected(nFirstTargetCol, nFirstTargetCol + nCols) )
	{
		if( IDNO == MessageBox(GetWindow(),
			_L("The data's target window has a locked column.\nDo you want to continue import?"),
			_L("File Import"), MB_YESNO) )
		{
			return IMPERR_CANCEL_ON_LOCKED_COL;
		}
	}
	/// end IMPORT_INTO_LOCKED_COL_WARNING
	
	/// EJP 09-15-2003 v7.5695 QA70-5135.19 AVOID_UNWANTED_REPEATING_OF_LAST_COL_DESIG_AND_FORMAT
	// When the col designation and format strings have less cols than the wks
	// we need to append the current existing settings to avoid having our
	// last setting repeating and overwriting the existing settings.
	if( strColDesig.GetLength() < wks.GetNumCols() && !bRepetitive )
	{
		string strTmp = wks.GetColFormats();
		strTmp.Delete(0, strColDesig.GetLength());
		strColFormat += strTmp;
		
		strTmp = wks.GetColDesignations();
		strTmp.Delete(0, strColDesig.GetLength());
		strColDesig += strTmp;
	}
	/// end AVOID_UNWANTED_REPEATING_OF_LAST_COL_DESIG_AND_FORMAT
	
	if( !strColDesig.IsEmpty() )
		wks.SetColDesignations(strColDesig, bRepetitive);
	if( !strColFormat.IsEmpty() )
		wks.SetColFormats(strColFormat, bRepetitive);

	return IMPERR_NONE;
}

//--------------------------------------------------------------------------
//--------------------------------------------------------------------------
static int HeaderVariablesToPageInfo(Page &pgTarget, TreeNode &trFilter, LPCSTR lpcszFile)
{
	if( !pgTarget || lpcszFile == NULL )
		return 1;
	
	pgTarget.Info.Add("User");
	pgTarget.Info.User.AddSection("Variables");
	
	StringArray saNames;
	StringArray saValues;
	GetHeaderVariableNamesAndValues(trFilter, saNames, saValues, lpcszFile);

	string str;
	
	using var = pgTarget.Info.User.Variables;    
	for( int n = 0; n < saNames.GetSize(); n++ )
		var.AddString(saNames[n], saValues[n]);

	return 0;
}

static int GetHeaderVariableNamesAndValues(TreeNode &trFilter, StringArray &saNames, StringArray &saValues, LPCSTR lpcszFile)
{
	if( FILTER_TYPE_ASCII != trFilter.Type.nVal && FILTER_TYPE_BINARY != trFilter.Type.nVal )
		return 1;

	saNames.SetSize(0);
	saValues.SetSize(0);

	string str;
	
	if( FILTER_TYPE_ASCII == trFilter.Type.nVal )
	{
		/// EJP 07-28-2003 v7.0637 QA70-4911 IMPROVE_HDR_VAR_SCAN
		/*
		int iFirstLine, iLastLine, iSeparator;
		if( fuGetHdrVarScan(trFilter, iFirstLine, iLastLine, iSeparator) )
		{
			StringArray saLines;
			ReadFileLines(saLines, lpcszFile, iLastLine - iFirstLine + 1, iFirstLine);

			int i;
			string strName, strValue;
			
			for( int iLine = 0; iLine < saLines.GetSize(); iLine++ )
			{
				i = saLines[iLine].Find(iSeparator);
				if( i > 0 )
				{
					strName = saLines[iLine].Left(i);
					strName.TrimLeft();
					strName.TrimRight();
					strName.MakeValidCName('_');
					if( !strName.IsEmpty() )
					{
						strValue = saLines[iLine].Mid(i + 1);
						strValue.TrimLeft();
						strValue.TrimRight();
						
						saNames.Add(strName);
						saValues.Add(strValue);
					}
				}
			}
		}
		*/
		get_ascii_file_header_variables(saNames, saValues, lpcszFile, trFilter);
		/// end IMPROVE_HDR_VAR_SCAN
	}
	else // FILTER_TYPE_BINARY
	{
		fuGetHeaderParamNames(trFilter, saNames);

		file fil;
		if( fil.Open(lpcszFile, file::modeRead|file::typeBinary) )
		{
			BINIMP binimp;
			fuGetBINIMP(trFilter, binimp);
			
			int nType, nOffset, nSize;
			for( int n = 0; n < saNames.GetSize(); n++ )
			{
				if( !fuGetBinHeaderParam(trFilter, saNames[n], nType, nOffset, nSize) )
					continue; // failed to get param
				if( ReadBinaryHeaderParam(str, fil, nType, nOffset, nSize, !binimp.iBigEndian) )
					saValues.Add(str);
			}
			fil.Close();
		}
	}
	return 0;
}

//--------------------------------------------------------------------------
/// EJP 07-28-2003 v7.0637 QA70-4911 IMPROVE_HDR_VAR_SCAN
static void check_and_add_header_variable(StringArray &saNames, StringArray &saValues, LPCSTR lpcszName, LPCSTR lpcszValue)
{
	string strName(lpcszName);
	strName.TrimLeft();
	strName.TrimRight();
	strName.MakeValidCName('_');
	if( !strName.IsEmpty() )
	{
		string strValue(lpcszValue);
		strValue.TrimLeft();
		strValue.TrimRight();

		saNames.Add(strName);
		saValues.Add(strValue);
	}
}

static void get_tokens_ignoring_quotes(StringArray &saTokens, LPCSTR lpcszTokens, int iSeparator)
{
	string strToken, strTokens = lpcszTokens;

	int i = strTokens.Find(iSeparator);
	while( i != -1 )
	{
		saTokens.Add(strTokens.Left(i)); // add everything before the separator into the string array
		strTokens.Delete(0, i + 1); // delete the separator and everything before it
		i = strTokens.Find(iSeparator);
	}
	saTokens.Add(strTokens);
}

void get_ascii_file_header_variables(StringArray &saNames, StringArray &saValues, LPCSTR lpcszFile, TreeNode &trFilter)
{
	int iFirstLine, iLastLine, iSeparator;
	if( !fuGetHdrVarScan(trFilter, iFirstLine, iLastLine, iSeparator) )
		return;
	
	StringArray saLines;
	ReadFileLines(saLines, lpcszFile, iLastLine - iFirstLine + 1, iFirstLine);

	int i, iLine;

	// First loop assumes each line is formatted as: <name> <separator> <value>
	for( iLine = 0; iLine < saLines.GetSize(); iLine++ )
	{
		i = saLines[iLine].Find(iSeparator);
		if( i > 0 )
			check_and_add_header_variable(saNames, saValues, saLines[iLine].Left(i), saLines[iLine].Mid(i + 1));
	}

	// Second loop assumes each line is formatted as: <value1> <separator> <value2> <separator> ... <valueN>
	string strName;
	StringArray saTokens;
	for( iLine = 0; iLine < saLines.GetSize(); iLine++ )
	{
		try
		{
			saLines[iLine].GetTokens(saTokens, iSeparator);
		}
		catch(int nErr)
		{
			///printf("Runtime error %d calling string::GetTokens while scanning ASCII header line %d.\n", nErr, iLine + 1);
			///continue;
			saTokens.SetSize(0);
			get_tokens_ignoring_quotes(saTokens, saLines[iLine], iSeparator);
		}
		for( int iToken = 0; iToken < saTokens.GetSize(); iToken++ )
		{
			strName.Format("L%dV%d", iFirstLine + iLine + 1, iToken + 1);
			check_and_add_header_variable(saNames, saValues, strName, saTokens[iToken]);
		}
	}
}
/// end IMPROVE_HDR_VAR_SCAN

//--------------------------------------------------------------------------
//--------------------------------------------------------------------------
int ImportASCII(Page &pgTarget, TreeNode &trFilter, LPCSTR lpcszFile, int nFile)
{
	if( trFilter.Type.nVal != FILTER_TYPE_ASCII )
		return 1;

	string str = pgTarget.GetName();
	Worksheet wks(str);
	if( !wks )
		return 1;
	
	ASCIMP ascimp;
	fuGetASCIMP(trFilter, ascimp);

	// An internal bug can appear when partial import is off and user specifies
	// a number of columns that differs from the number auto-detected.
	// A fix is to do a partial import but set the partial import settings
	// to import all the data.
	if( ascimp.iPartial == 0 )
	{
		ascimp.iPartial=1;
		ascimp.iPartialC1 = 0;
		ascimp.iPartialC2 = ascimp.iNumColumns - 1;
		ascimp.iPartialR1 = 0;
		ascimp.iPartialR2 = -1;
	}

	int nErr = wks.ImportASCII(lpcszFile, ascimp);
	if( nErr )
		return nErr;

	/// EJP 11-12-2003 v7.5753 QA70-5211.11 FIX_SETTING_OF_DESIG_AND_FORMAT
	/* This is being done in ImportFile which knows about appending new cols
	str = fuGetDesignations(trFilter);
	if( !str.IsEmpty() )
		wks.SetColDesignations(str, fuGetRepetitive(trFilter));
	str = fuGetFormats(trFilter);
	if( !str.IsEmpty() )
		wks.SetColFormats(str, fuGetRepetitive(trFilter));
	*/
	/// end FIX_SETTING_OF_DESIG_AND_FORMAT

	return 0; // success
}

//--------------------------------------------------------------------------
//--------------------------------------------------------------------------
int ImportBinary(Page &pgTarget, TreeNode &trFilter, LPCSTR lpcszFile, int nFile)
{
	if( trFilter.Type.nVal != FILTER_TYPE_BINARY )
		return 1;

	string str = pgTarget.GetName();
	Worksheet wks(str);
	if( !wks )
		return 1;
	
	BINIMP binimp;
	fuGetBINIMP(trFilter, binimp);

	int nErr = wuImportBinary(wks, lpcszFile, &binimp);
	if( nErr )
		return nErr;
	
	/// EJP 11-12-2003 v7.5753 QA70-5211.11 FIX_SETTING_OF_DESIG_AND_FORMAT
	/* This is being done in ImportFile which knows about appending new cols
	str = fuGetDesignations(trFilter);
	if( !str.IsEmpty() )
		wks.SetColDesignations(str, fuGetRepetitive(trFilter));
	str = fuGetFormats(trFilter);
	if( !str.IsEmpty() )
		wks.SetColFormats(str, fuGetRepetitive(trFilter));
	*/
	/// end FIX_SETTING_OF_DESIG_AND_FORMAT

	return 0; // success
}

#ifdef _OPERATION_H_WAIT_FIX
bool BatchProcessing()
{
	PageBase pbActive;
	pbActive = Project.Pages();
	if( !pbActive || EXIST_WKS != pbActive.GetType() )
		return false;

	if( false ) // if wks is not setup for batch processing
	{
		// these strings are just temporary.  they will be localized later when checking for batch processing is implemented
		MessageBox(GetWindow(), "The current worksheet is not setup for Batch Processing.", "Batch Processing");
		return false;
	}

	string strPgName = pbActive.GetName();
	Page pgTarget(strPgName);
	if( !pgTarget )
		return false;
	
	FDLogInit();

	// I'm doing all the FDlog stuff here because I need control that I can
	// not get with the functions in sys_utils.
	
	using fdlog = LabTalk.FDlog;

	fdlog.NumTypes = 0; // remove all file types
	fdlog.UseGroup("ASCII");
	fdlog.AddUserTypes(); // add file types from filters
	
	fdlog.MultiOpen.ColView = 12; // 4(file size) + 8(date)
	fdlog.MultiOpen.Sort = 0; // 0=standard sorting, 1=group sorting
	fdlog.MultiOpen.ComboName$ = "Import Filter";

	// Get filters from Origin and UserFiles folders
	StringArray saFilterFiles;
	fuGetFilterFiles(saFilterFiles);

	// Get a list of filters for displaying to users
	StringArray saFilterList;
	fuGetFilterList(saFilterList, saFilterFiles);

	// Put the display list in the file dialog's combobox
	string str;
	str.SetTokens(saFilterList, '|');
	fdlog.MultiOpen.ComboList$ = str;

	// Prompt the user with a file dialog
	if( NANUM == fdlog.MultiOpen() )
		return false; // no files selected
	
	int i = fdlog.MultiOpen.ComboSel - 1; // -1 for LabTalk's 1-based index
	Tree trFilter;
	if( !trFilter.Load(saFilterFiles[i]) )
		return false; // load filter error
	pgTarget.Info.System.Import.Filter$ = saFilterFiles[i];

	Project.Operations.SetBatchProcessing();

	char sz[MAXFULLPATH];
	for( i = 0; i < fdlog.MultiOpen.Count; i++ )
	{
		// Get data file
		fdlog.Get("A", i + 1); // +1 for LabTalk's 1-based index
		LT_get_str("%A", sz, MAXFULLPATH);
		
		// Import data file
		ImportFile(pgTarget, trFilter, sz, i);
		Project.Operations.Run();
	}

	Project.Operations.SetBatchProcessing(FALSE);
	return true;
}
#endif //_OPERATION_H

/// EJP 09-24-2003 v7.5706 QA70-5230 NOTIFY_USER_OF_IMPORT_ERROR
int ImportErrorMsgBox(int iErr, LPCSTR lpcszFilter)
{
	string str, strErr;
	bool bAppendFilterName = false;

	switch( iErr )
	{
	case IMPERR_NO_FILTER:
		strErr = _L("No filter specified.");
		break;
	case IMPERR_LOAD_FILTER:
		str = _L("Error loading import filter:\n%s");
		strErr.Format(str, lpcszFilter);
		break;
	case IMPERR_NO_TARGET_PAGE:
		strErr = _L("No target page for data.");
		bAppendFilterName = true;
		break;
	case IMPERR_GET_IMPORT_FUNC:
		string strFilterType;
		Tree trFilter;
		if( lpcszFilter && trFilter.Load(lpcszFilter) )
		{
			fuGetFilterType(trFilter, strFilterType);
			strErr.Format(_L("Failed to get pointer to %s import function."), strFilterType);
		}
		else
			strErr = _L("Failed to get pointer to import function.");
		break;
	case IMPERR_IMPORT_FUNC_ERR:
		string strFilterType;
		Tree trFilter;
		if( lpcszFilter && trFilter.Load(lpcszFilter) )
		{
			fuGetFilterType(trFilter, strFilterType);
			strErr.Format(_L("%s import function returned error."), strFilterType);
		}
		else
			strErr = _L("The import function returned error.");
		bAppendFilterName = true;
		break;
	case IMPERR_PREPARE_WKS:
		strErr = _L("Error preparing worksheet for import.");
		bAppendFilterName = true;
		break;
	case IMPERR_FILTER_TYPE:
		strErr = _L("Unknown filter type.");
		bAppendFilterName = true;
		break;
	case IMPERR_NONE:
	case IMPERR_CANCEL_ON_LOCKED_COL:
		return 0; // no error to report
	default:
		strErr = _L("Unknown error.");
		bAppendFilterName = true;
		break;
	}

	string strMsg;
	if( bAppendFilterName && lpcszFilter )
		strMsg.Format(_L("%s\n\nUsing filter:\n%s"), strErr, lpcszFilter);
	else
		strMsg = strErr;
	MessageBox(GetWindow(), strMsg, _L("File Import Error"));

	return 0;
}
/// end NOTIFY_USER_OF_IMPORT_ERROR
