Here is my code, basically one the 4 computer I have tested it on they all work perfectly with very large data sizes, eg textfiles up to 500mb in size, but when I run them on the server with real data even files as small as 6mb seem to overrun somewhere and writes garbage to the end of my files.<\/p>\n
Here is the source of the entire function so people can have a more indepth look<\/p>\n
\/** Reads values from tagname between start_time and end_time which are strings in the format \n of 01\/01\/1970-12:00, a null string is treated as 01\/01\/1970, values are stored in\n \"tagname\".csv *\/\nint ReadValues(char * tagname, char * start_time, char * end_time, char * working_directory, char * new_tag_name)\n{\n long lRet;\n int number_of_samples;\n int loop;\n IHU_DATA_SAMPLE * pSamples=NULL;\n IHU_TIMESTAMP StartTime;\n IHU_TIMESTAMP EndTime;\n FILE *stream;\n char outputFileName[200];\n char szQuality[100];\n char newTempTagName[100];\n int Year;\n int Month;\n int Day;\n int Hour;\n int Minute;\n int Second;\n long Subsecond;\n int date[10];\n\n \/\/if we want to change the tagname do it now\n if(new_tag_name != 0){\n strncpy(newTempTagName, new_tag_name, 100);\n } else {\n strncpy(newTempTagName, tagname, 100); \n } \n\n \/\/ if the tagname contains a character that is invalid for a filename then we have to make a name\n if ( (strstr(newTempTagName, \"\/\")) || (strstr(newTempTagName, \"\\\\\")) || (strstr(newTempTagName, \":\")))\n {\n\n sprintf(outputFileName, \"MadeUpTag%d\", MadeUpTagCount++);\n }\n else\n {\n\n \/\/If a working directory was passed in use it\n if(!working_directory){ \n strncpy(outputFileName, newTempTagName,199); \n } else {\n strcpy(outputFileName, working_directory); \/\/Copy the working directory to the filename\n \/\/ Append a \/ on the end of the working directory if it doesn't already have one\n if(outputFileName[strlen(outputFileName)-1] == '\\\"'){ \n outputFileName[strlen(outputFileName)-1] = 0;\n } \n if(outputFileName[strlen(outputFileName)-1] != '\\\\'){ \n strncat(outputFileName, \"\\\\\", 199); \n }\n strncat(outputFileName, newTempTagName, 199); \/\/Copy the tagname to the end of the working directory\n }\n\n }\n\n \/\/Add the csv file extension\n strcat(outputFileName, \".csv\");\n #ifdef DEBUG\n printf(\"Output filename: %s\\n\", outputFileName);\n #endif\n\n stream = fopen(outputFileName, \"w\");\n if( stream == NULL ) {\n printf(\"The file %s can not be opened\\n\", outputFileName );\n } else {\n\n \/\/If start_time == 0 we want to start at 1970\n if(start_time == 0){\n \/\/ we want all the data so use an old start time\n struct tm local;\n local.tm_year = date[2] - 1900;\n local.tm_mon = date[1] - 1;\n local.tm_mday = date[0];\n local.tm_hour = date[3];\n local.tm_min = date[4];\n local.tm_sec = 0;\n time_t utc_seconds = mktime(&local);\n StartTime.Seconds = (long)utc_seconds;\n StartTime.Subseconds = 0; \n\n } else {\n \/\/we have been supplied a start time\n #ifdef DEBUG\n printf(\"Start Time: \");\n #endif\n lRet = convert_date(start_time, &StartTime);\n #ifdef DEBUG\n\n printf(\"Seconds %ld \\n\", StartTime.Seconds);\n #endif\n }\n\n \/\/if end_time == 0 we want to go all the way until now.\n if(end_time == 0){\n \/\/ end time of 0 means now\n memset(&EndTime, 0, sizeof(EndTime));\n } else {\n \/\/we have been supplied an end time\n #ifdef DEBUG\n printf(\"End Time: \");\n #endif\n lRet = convert_date(end_time, &EndTime);\n #ifdef DEBUG\n printf(\"Seconds %ld \\n\", EndTime.Seconds);\n #endif\n }\n\n \/\/ API will determine actual samples that are in that time range\n number_of_samples=0;\n\n \/\/ API will allocate the memory\n pSamples = NULL;\n \/\/timeTaken();\n lRet = ihuReadRawDataByTime\n (\n serverhandle, \/\/ the handle returned from the connect\n tagname, \/\/ the single tagname to retrieve \n &StartTime, \/\/ start time for query\n &EndTime, \/\/ end time for query\n &number_of_samples, \/\/ will be set by API\n &pSamples \/\/ will be allocated and populated in the user API\n );\n\n char temp[100];\n char Header[100];\n char lengthOfHeader = 0;\n char SampleAndTag[100];\n int lengthOfSampleAndTag = 0;\n char ActualSample[100];\n int lengthOfActualSample = 0;\n char NumberOfSamples[100];\n int lengthOfNumberOfSamples = 0;\n int oldQualityStatus = 99999;\n\n if (lRet == ihuSTATUS_OK)\n { \n\n \/\/ successful read\n lengthOfHeader = snprintf(Header, 100, \"[Data]\\nTagname,TimeStamp,Value,DataQuality\\n\");\n if(lengthOfHeader < 0){\n printf(\"ERROR WRITING TO BUFFER!\\n\"); \n } else {\n if(fwrite(Header, 1, lengthOfHeader, stream) < lengthOfHeader){\n printf(\"ERROR WRITING TO FILE!\\n\");\n }\n }\n\n for (loop = 0;loop < number_of_samples;loop++)\n {\n struct tm * local;\n local = localtime(&pSamples[loop].TimeStamp.Seconds);\n Month = local->tm_mon + 1;\n Day = local->tm_mday;\n Year = local->tm_year + 1900;\n Hour = local->tm_hour;\n Minute = local->tm_min;\n Second = local->tm_sec;\n Subsecond = pSamples[loop].TimeStamp.Subseconds;\n\n \/\/lengthOfSampleAndTag = snprintf(SampleAndTag, 100, \"Sample %d, %s\",loop, newTempTagName);\n lengthOfSampleAndTag = snprintf(SampleAndTag, 100, \"%s\", newTempTagName);\n if(lengthOfSampleAndTag < 0){\n printf(\"ERROR WRITING TO BUFFER!\\n\"); \n } else {\n if(fwrite(SampleAndTag, 1,lengthOfSampleAndTag, stream) < lengthOfSampleAndTag){\n printf(\"ERROR WRITING TO FILE!\\n\");\n }\n }\n\n \/\/Doing the formatting ourselves manually per character saves about 20% of cpu time\n \/\/on large databases this can save hours\n temp[0] = ',';\n temp[1] = ' ';\n temp[2] = Month\/10 + 0x30; \/\/Tens\n temp[3] = Month%10 + 0x30; \/\/units\n temp[4] = '\/';\n temp[5] = Day\/10 + 0x30;\n temp[6] = Day%10 + 0x30;\n temp[7] = '\/';\n temp[8] = Year\/1000 + 0x30;\n temp[9] = '0';\n temp[10] = (Year%100)\/10 + 0x30;\n temp[11] = (Year%100)%10 + 0x30;\n temp[12] = ' ';\n temp[13] = Hour\/10 + 0x30; \/\/Tens\n temp[14] = Hour%10 + 0x30; \/\/units\n temp[15] = ':';\n temp[16] = Minute\/10 + 0x30;\n temp[17] = Minute%10 + 0x30;\n temp[18] = ':';\n temp[19] = Second\/10 + 0x30;\n temp[20] = Second%10 + 0x30;\n temp[21] = '.';\n temp[22] = '0';\n temp[23] = '0';\n temp[24] = '0';\n temp[25] = ',';\n temp[26] = ' ';\n temp[27] = 0; \/\/Null termination\n\n \/\/This is to save copying the string if it is the same\n if(oldQualityStatus != pSamples[loop].Quality.QualityStatus){\n oldQualityStatus = pSamples[loop].Quality.QualityStatus;\n switch(pSamples[loop].Quality.QualityStatus)\n {\n case ihuOPCBad:\n strncpy(szQuality,\"Bad\",99);\n break;\n case ihuOPCUncertain:\n strncpy(szQuality,\"Uncertain\",99);\n break;\n case ihuOPCNA:\n strncpy(szQuality,\"NotAvailable\",99);\n break;\n case ihuOPCGood:\n strncpy(szQuality,\"Good\",99);\n break;\n default:\n strncpy(szQuality,\"Really unknown\",99);\n break;\n }\n }\n\n if ( pSamples[loop].ValueDataType == ihuFloat )\n {\n \/\/lengthOfActualSample = snprintf(ActualSample, 100,\"%s%8.7f, %s, Type:Float\\n\",temp,pSamples[loop].Value.Float,szQuality);\n lengthOfActualSample = snprintf(ActualSample, 100,\"%s%8.7f, %s\\n\",temp,pSamples[loop].Value.Float,szQuality);\n if(lengthOfActualSample < 0){\n printf(\"ERROR WRITING TO BUFFER!\\n\"); \n } else {\n if(fwrite(ActualSample, 1, lengthOfActualSample, stream) < lengthOfActualSample){\n printf(\"ERROR WRITING TO FILE!\\n\");\n }\n }\n }\n else if ( pSamples[loop].ValueDataType == ihuDoubleFloat )\n {\n \/\/lengthOfActualSample = snprintf(ActualSample, 100,\"%s%8.15f, %s, Type: DoubleFloat\\n \",temp,pSamples[loop].Value.DoubleFloat,szQuality);\n lengthOfActualSample = snprintf(ActualSample, 100,\"%s%8.15f, %s\\n \",temp,pSamples[loop].Value.DoubleFloat,szQuality);\n if(lengthOfActualSample < 0){\n printf(\"ERROR WRITING TO BUFFER!\\n\"); \n } else {\n if(fwrite(ActualSample, 1,lengthOfActualSample, stream) < lengthOfActualSample){\n printf(\"ERROR WRITING TO FILE!\\n\");\n }\n }\n\n }\n else if ( pSamples[loop].ValueDataType == ihuShort )\n {\n \/\/lengthOfActualSample = snprintf(ActualSample, 100,\"%s%d, %s, Type:Short\\n\",temp,pSamples[loop].Value.Short,szQuality);\n lengthOfActualSample = snprintf(ActualSample, 100,\"%s%d, %s\\n\",temp,pSamples[loop].Value.Short,szQuality);\n if(lengthOfActualSample<\/code><\/pre>\nOriginally posted 2013-11-09 23:23:03. <\/small><\/p>","protected":false},"excerpt":{"rendered":"
Here is my code, basically one the 4 computer I have tested it on they all work perfectly with very large data sizes, eg textfiles up to 500mb in size, but when I run them on the server with real data even files as small as 6mb seem to overrun somewhere and writes garbage to […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1094","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"_links":{"self":[{"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/posts\/1094","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/comments?post=1094"}],"version-history":[{"count":0,"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/posts\/1094\/revisions"}],"wp:attachment":[{"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/media?parent=1094"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/categories?post=1094"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/unknownerror.org\/index.php\/wp-json\/wp\/v2\/tags?post=1094"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}