CSV to Tick import problem

Posted By: Fiber

CSV to Tick import problem - 08/17/17 09:17

Hi,
I downloaded script CSVtoT6andT1.c from the forum and trying to import tick data from CSV.
After running import, only every second line is imported from CSV file, though Zorro messages states that all lines are processed. What could be a problem?

Code:
// convert price history from .csv to .t1 or .t6

#define TRADESTATION
//#define HISTDATA
//#define YAHOO // convert csv data from R or Yahoo

#ifdef TRADESTATION
#define TCK T1
string InName = "History\ticks.txt";  // name of the CSV file
string OutName = "History\ticks.t1";
#endif
#ifdef HISTDATA
#define TCK T6
string InName = "History\DAT_ASCII_USDZAR_M1_2015.csv";  // name of the CSV file
string OutName = "History\USDZAR_2015hd.t6";
#endif
#ifdef YAHOO
#define TCK T6
string InName = "History\table.csv";  // name of the CSV file
string OutName = "History\Stock.t6";
#endif

typedef struct _SYSTEMTIME {
    WORD wYear;
    WORD wMonth;
    WORD wDayOfWeek;
    WORD wDay;
    WORD wHour;
    WORD wMinute;
    WORD wSecond;
    WORD wMilliseconds;
} SYSTEMTIME;

API(SystemTimeToVariantTime,oleaut32)
int _stdcall SystemTimeToVariantTime(SYSTEMTIME* lpSystemTime, double* pvtime);

DATE ConvertTime(int Year,int Month,int Dom,int Hour,int Minute,int Second)
{
	SYSTEMTIME Time;
	memset(&Time,0,sizeof(SYSTEMTIME));
	if(Year < 50) Year += 2000;
	else if(Year < 100) year += 1900;
	Time.wYear = Year;
	Time.wMonth = Month;
	Time.wDay = Dom;
	Time.wHour = Hour;
	Time.wMinute = Minute;
	Time.wSecond = Second;
	
	DATE vTime;
	SystemTimeToVariantTime(&Time,&vTime);
#ifdef HISTDATA
	vTime += 5./24.; // add 5 hours for converting EST to UTC
#endif
	return vTime;
}

string readTick(string content,TCK* tick)
{
// tokenize a single line	
	char* line = strtok(content,"n");
	if(!line) return 0;

	int Year, Month, Dom, Hour = 0, Minute = 0, Second = 0; 
#ifdef HISTDATA // line format "20100103 170000;1.430100;1.430400;1.430100;1.430400;0"

	if(10 != sscanf(line,"%4d%2d%2d %2d%2d%2d;%f;%f;%f;%f;",
		&Year, &Month, &Dom, &Hour, &Minute, &Second,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose)) 
		return 0;
	tick->fVal = tick->fVol = 0;
#endif
#ifdef YAHOO // line format "2015-05-29,43.45,43.59,42.81,42.94,10901500,42.94"
	float fAdj;

	if(9 != sscanf(line,"%4d-%2d-%2d,%f,%f,%f,%f,%f,%f",
		&Year, &Month, &Dom,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose,&tick->fVol,&fAdj)) 
		return 0;
	if(tick->fClose > 0. && fAdj > 0.) { // adjust for dividends and splits
		float f = fAdj/tick->fClose;
		tick->fOpen *= f;
		tick->fHigh *= f;
		tick->fLow *= f;
		tick->fClose = fAdj;
		tick->fVal = 0;
	}
#endif
#ifdef TRADESTATION // line format "06/30/2016,17:00:00,2086.50,2086.50,2086.50,2086.50,319,0"

	if(7 != sscanf(line,"%2d/%2d/%4d,%2d:%2d:%2d,%f",
		&Month, &Dom, &Year, &Hour, &Minute, &Second,
		&tick->fVal)) 
		return 0;
#endif

// store the time in DATE format
	tick->time = ConvertTime(Year,Month,Dom,Hour,Minute,Second);

// return pointer to next line
	return line+strlen(line)+1;
}

function main()
{
	if(!file_length(InName)) {
		quit("Data file not found!");
		return;
	}
	string content = file_content(InName);
		
// allocate TICK array		
	int maxticks = strcount(content,0x0a); 
	printf("n%d lines processing...",maxticks);
	int skipped = 0;
	TCK* ticks = malloc(maxticks*sizeof(TCK));

#ifdef HISTDATA
// read ticks in reverse order	
	TCK* tick = ticks+maxticks; 
	while(content)
		content = readTick(content,--tick);

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif	

#ifdef YAHOO
// skip the header
	content = strstr(content,"n")+1;
// already contains ticks in reverse order	
	TCK* tick = ticks; 
	while(content) {
		content = readTick(content,tick++);
	}
// store the ticks
	int size = (int)(tick-1)-(int)(ticks);
	file_write(OutName,ticks,size);
#endif

#ifdef TRADESTATION
// skip the header
	content = strstr(content,"n")+1;
// read ticks in reverse order	
	TCK* tick = ticks+maxticks;
	while(content) {
		content = readTick(content,--tick);
		if(tick < ticks+maxticks-1) {
			TCK* prev = tick+1;
			if(tick->fVal == prev->fVal) {
				++tick; // skip unchanged quotes
				++skipped;
			} else if(tick->time <= prev->time)
				tick->time += 1./(24.*60.*60.*100.); // add 10 ms
		}	
	}

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif

	free(ticks);
	printf("n%d ticks written",size/sizeof(TCK));
	if(skipped)
		printf(", %d skipped",skipped);
}



Attached picture CsvTickImport.jpg
Posted By: pcz

Re: CSV to Tick import problem - 08/17/17 14:40

Fiber: If I remember correctly the ZHistoryEditor is not showing all the records properly. You can try to export the data yourself and you'll see if it's the case.
Posted By: Fiber

Re: CSV to Tick import problem - 08/17/17 18:38

Thanks pcz!
You are right, after exporting I can see that all tick data is in the file.
© 2024 lite-C Forums