// convert price history from .csv to .t1 or .t6

//#define TRADESTATION
//#define HISTDATA
//#define YAHOO // convert csv data from R or Yahoo
#define CSVEXPORT

#ifdef TRADESTATION
#define TCK T1
string InName = "History\ticks.txt";  // name of the CSV file
string OutName = "History\ticks.t1";
#endif
#ifdef HISTDATA
#define TCK T6
string InName = "History\DAT_ASCII_USDZAR_M1_2015.csv";  // name of the CSV file
string OutName = "History\USDZAR_2015hd.t6";
#endif
#ifdef YAHOO
#define TCK T6
string InName = "History\table.csv";  // name of the CSV file
string OutName = "History\Stock.t6";
#endif
#ifdef CSVEXPORT
#define TCK T6
string InName = "History\GBPJPY.csv"; // name of the CSV file
string OutName = "History\GBPJPY2009.t6";
#endif

typedef struct _SYSTEMTIME {
    WORD wYear;
    WORD wMonth;
    WORD wDayOfWeek;
    WORD wDay;
    WORD wHour;
    WORD wMinute;
    WORD wSecond;
    WORD wMilliseconds;
} SYSTEMTIME;

API(SystemTimeToVariantTime,oleaut32)
int _stdcall SystemTimeToVariantTime(SYSTEMTIME* lpSystemTime, double* pvtime);

DATE ConvertTime(int Year,int Month,int Dom,int Hour,int Minute,int Second)
{
	SYSTEMTIME Time;
	memset(&Time,0,sizeof(SYSTEMTIME));
	if(Year < 50) Year += 2000;
	else if(Year < 100) year += 1900;
	Time.wYear = Year;
	Time.wMonth = Month;
	Time.wDay = Dom;
	Time.wHour = Hour;
	Time.wMinute = Minute;
	Time.wSecond = Second;
	
	DATE vTime;
	SystemTimeToVariantTime(&Time,&vTime);
#ifdef HISTDATA
	vTime += 5./24.; // add 5 hours for converting EST to UTC
#endif
	return vTime;
}

string readTick(string content,TCK* tick)
{
// tokenize a single line	
	char* line = strtok(content,"n");
	if(!line) return 0;

	int Year, Month, Dom, Hour = 0, Minute = 0, Second = 0; 
#ifdef HISTDATA // line format "20100103 170000;1.430100;1.430400;1.430100;1.430400;0"

	if(10 != sscanf(line,"%4d%2d%2d %2d%2d%2d;%f;%f;%f;%f;",
		&Year, &Month, &Dom, &Hour, &Minute, &Second,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose)) 
		return 0;
	tick->fVal = tick->fVol = 0;
#endif
#ifdef YAHOO // line format "2015-05-29,43.45,43.59,42.81,42.94,10901500,42.94"
	float fAdj;

	if(9 != sscanf(line,"%4d-%2d-%2d,%f,%f,%f,%f,%f,%f",
		&Year, &Month, &Dom,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose,&tick->fVol,&fAdj)) 
		return 0;
	if(tick->fClose > 0. && fAdj > 0.) { // adjust for dividends and splits
		float f = fAdj/tick->fClose;
		tick->fOpen *= f;
		tick->fHigh *= f;
		tick->fLow *= f;
		tick->fClose = fAdj;
		tick->fVal = 0;
	}
#endif
#ifdef TRADESTATION // line format "06/30/2016,17:00:00,2086.50,2086.50,2086.50,2086.50,319,0"

	if(7 != sscanf(line,"%2d/%2d/%4d,%2d:%2d:%2d,%f",
		&Month, &Dom, &Year, &Hour, &Minute, &Second,
		&tick->fVal)) 
		return 0;
#endif
#ifdef CSVEXPORT // line format "2009-11-16 14:00, 149.78300, 149.92300, 149.51500, 149.61800"
	if(9 != sscanf(line,"%4d-%2d-%2d %2d:%2d, %f, %f, %f, %f",
		&Year, &Month, &Dom, &Hour, &Minute, &Second,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose)) 
		return 0;
	tick->fVal = tick->fVol = 0;
#endif

// store the time in DATE format
	tick->time = ConvertTime(Year,Month,Dom,Hour,Minute,Second);

// return pointer to next line
	return line+strlen(line)+1;
}

function main()
{
	if(!file_length(InName)) {
		quit("Data file not found!");
		return;
	}
	string content = file_content(InName);
		
// allocate TICK array		
	int maxticks = strcount(content,0x0a); 
	printf("n%d lines processing...",maxticks);
	int skipped = 0;
	TCK* ticks = malloc(maxticks*sizeof(TCK));

#ifdef HISTDATA
// read ticks in reverse order	
	TCK* tick = ticks+maxticks; 
	while(content)
		content = readTick(content,--tick);

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif	

#ifdef YAHOO
// skip the header
	content = strstr(content,"n")+1;
// already contains ticks in reverse order	
	TCK* tick = ticks; 
	while(content) {
		content = readTick(content,tick++);
	}
// store the ticks
	int size = (int)(tick-1)-(int)(ticks);
	file_write(OutName,ticks,size);
#endif

#ifdef TRADESTATION
// skip the header
	content = strstr(content,"n")+1;
// read ticks in reverse order	
	TCK* tick = ticks+maxticks;
	while(content) {
		content = readTick(content,--tick);
		if(tick < ticks+maxticks-1) {
			TCK* prev = tick+1;
			if(tick->fVal == prev->fVal) {
				++tick; // skip unchanged quotes
				++skipped;
			} else if(tick->time <= prev->time)
				tick->time += 1./(24.*60.*60.*100.); // add 10 ms
		}	
	}

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif

#ifdef CSVEXPORT
// skip the header
	content = strstr(content,"n")+1;
// read ticks in reverse order	
	TCK* tick = ticks+maxticks; 
	while(content)
		content = readTick(content,--tick);

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif

	free(ticks);
	printf("n%d ticks written",size/sizeof(TCK));
	if(skipped)
		printf(", %d skipped",skipped);
}