Gamestudio Links
Zorro Links
Newest Posts
Blobsculptor tools and objects download here
by NeoDumont. 03/28/24 03:01
Issue with Multi-Core WFO Training
by aliswee. 03/24/24 20:20
Why Zorro supports up to 72 cores?
by Edgar_Herrera. 03/23/24 21:41
Zorro Trader GPT
by TipmyPip. 03/06/24 09:27
VSCode instead of SED
by 3run. 03/01/24 19:06
AUM Magazine
Latest Screens
The Bible Game
A psychological thriller game
SHADOW (2014)
DEAD TASTE
Who's Online Now
2 registered members (Imhotep, opm), 785 guests, and 4 spiders.
Key: Admin, Global Mod, Mod
Newest Members
sakolin, rajesh7827, juergen_wue, NITRO_FOREVER, jack0roses
19043 Registered Users
Previous Thread
Next Thread
Print Thread
Rate Thread
CSV to Tick import problem #467616
08/17/17 09:17
08/17/17 09:17
Joined: Dec 2016
Posts: 43
F
Fiber Offline OP
Newbie
Fiber  Offline OP
Newbie
F

Joined: Dec 2016
Posts: 43
Hi,
I downloaded script CSVtoT6andT1.c from the forum and trying to import tick data from CSV.
After running import, only every second line is imported from CSV file, though Zorro messages states that all lines are processed. What could be a problem?

Code:
// convert price history from .csv to .t1 or .t6

#define TRADESTATION
//#define HISTDATA
//#define YAHOO // convert csv data from R or Yahoo

#ifdef TRADESTATION
#define TCK T1
string InName = "History\ticks.txt";  // name of the CSV file
string OutName = "History\ticks.t1";
#endif
#ifdef HISTDATA
#define TCK T6
string InName = "History\DAT_ASCII_USDZAR_M1_2015.csv";  // name of the CSV file
string OutName = "History\USDZAR_2015hd.t6";
#endif
#ifdef YAHOO
#define TCK T6
string InName = "History\table.csv";  // name of the CSV file
string OutName = "History\Stock.t6";
#endif

typedef struct _SYSTEMTIME {
    WORD wYear;
    WORD wMonth;
    WORD wDayOfWeek;
    WORD wDay;
    WORD wHour;
    WORD wMinute;
    WORD wSecond;
    WORD wMilliseconds;
} SYSTEMTIME;

API(SystemTimeToVariantTime,oleaut32)
int _stdcall SystemTimeToVariantTime(SYSTEMTIME* lpSystemTime, double* pvtime);

DATE ConvertTime(int Year,int Month,int Dom,int Hour,int Minute,int Second)
{
	SYSTEMTIME Time;
	memset(&Time,0,sizeof(SYSTEMTIME));
	if(Year < 50) Year += 2000;
	else if(Year < 100) year += 1900;
	Time.wYear = Year;
	Time.wMonth = Month;
	Time.wDay = Dom;
	Time.wHour = Hour;
	Time.wMinute = Minute;
	Time.wSecond = Second;
	
	DATE vTime;
	SystemTimeToVariantTime(&Time,&vTime);
#ifdef HISTDATA
	vTime += 5./24.; // add 5 hours for converting EST to UTC
#endif
	return vTime;
}

string readTick(string content,TCK* tick)
{
// tokenize a single line	
	char* line = strtok(content,"n");
	if(!line) return 0;

	int Year, Month, Dom, Hour = 0, Minute = 0, Second = 0; 
#ifdef HISTDATA // line format "20100103 170000;1.430100;1.430400;1.430100;1.430400;0"

	if(10 != sscanf(line,"%4d%2d%2d %2d%2d%2d;%f;%f;%f;%f;",
		&Year, &Month, &Dom, &Hour, &Minute, &Second,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose)) 
		return 0;
	tick->fVal = tick->fVol = 0;
#endif
#ifdef YAHOO // line format "2015-05-29,43.45,43.59,42.81,42.94,10901500,42.94"
	float fAdj;

	if(9 != sscanf(line,"%4d-%2d-%2d,%f,%f,%f,%f,%f,%f",
		&Year, &Month, &Dom,
		&tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose,&tick->fVol,&fAdj)) 
		return 0;
	if(tick->fClose > 0. && fAdj > 0.) { // adjust for dividends and splits
		float f = fAdj/tick->fClose;
		tick->fOpen *= f;
		tick->fHigh *= f;
		tick->fLow *= f;
		tick->fClose = fAdj;
		tick->fVal = 0;
	}
#endif
#ifdef TRADESTATION // line format "06/30/2016,17:00:00,2086.50,2086.50,2086.50,2086.50,319,0"

	if(7 != sscanf(line,"%2d/%2d/%4d,%2d:%2d:%2d,%f",
		&Month, &Dom, &Year, &Hour, &Minute, &Second,
		&tick->fVal)) 
		return 0;
#endif

// store the time in DATE format
	tick->time = ConvertTime(Year,Month,Dom,Hour,Minute,Second);

// return pointer to next line
	return line+strlen(line)+1;
}

function main()
{
	if(!file_length(InName)) {
		quit("Data file not found!");
		return;
	}
	string content = file_content(InName);
		
// allocate TICK array		
	int maxticks = strcount(content,0x0a); 
	printf("n%d lines processing...",maxticks);
	int skipped = 0;
	TCK* ticks = malloc(maxticks*sizeof(TCK));

#ifdef HISTDATA
// read ticks in reverse order	
	TCK* tick = ticks+maxticks; 
	while(content)
		content = readTick(content,--tick);

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif	

#ifdef YAHOO
// skip the header
	content = strstr(content,"n")+1;
// already contains ticks in reverse order	
	TCK* tick = ticks; 
	while(content) {
		content = readTick(content,tick++);
	}
// store the ticks
	int size = (int)(tick-1)-(int)(ticks);
	file_write(OutName,ticks,size);
#endif

#ifdef TRADESTATION
// skip the header
	content = strstr(content,"n")+1;
// read ticks in reverse order	
	TCK* tick = ticks+maxticks;
	while(content) {
		content = readTick(content,--tick);
		if(tick < ticks+maxticks-1) {
			TCK* prev = tick+1;
			if(tick->fVal == prev->fVal) {
				++tick; // skip unchanged quotes
				++skipped;
			} else if(tick->time <= prev->time)
				tick->time += 1./(24.*60.*60.*100.); // add 10 ms
		}	
	}

// store the ticks
	int size = (int)(ticks+maxticks)-(int)(tick+1);
	file_write(OutName,tick+1,size);
#endif

	free(ticks);
	printf("n%d ticks written",size/sizeof(TCK));
	if(skipped)
		printf(", %d skipped",skipped);
}


Attached Files
CsvTickImport.jpg (15 downloads)
Re: CSV to Tick import problem [Re: Fiber] #467622
08/17/17 14:40
08/17/17 14:40
Joined: May 2016
Posts: 180
Prague
pcz Offline
Member
pcz  Offline
Member

Joined: May 2016
Posts: 180
Prague
Fiber: If I remember correctly the ZHistoryEditor is not showing all the records properly. You can try to export the data yourself and you'll see if it's the case.

Re: CSV to Tick import problem [Re: pcz] #467625
08/17/17 18:38
08/17/17 18:38
Joined: Dec 2016
Posts: 43
F
Fiber Offline OP
Newbie
Fiber  Offline OP
Newbie
F

Joined: Dec 2016
Posts: 43
Thanks pcz!
You are right, after exporting I can see that all tick data is in the file.


Moderated by  Petra 

Gamestudio download | chip programmers | Zorro platform | shop | Data Protection Policy

oP group Germany GmbH | Birkenstr. 25-27 | 63549 Ronneburg / Germany | info (at) opgroup.de

Powered by UBB.threads™ PHP Forum Software 7.7.1