Hi, I am making connection with stock market live data provider through web sockets. I am getting nearly 40,000 records per minute in response for all companies live price. I need to store all that data into text file and so that I need to apply my code
for all responses and when I write code for all responses then it makes process very slow. When I apply code on response then it execute only 5,000 records. So it's very slow as compare to our responses. So how to make our process fast so I can store data
with receiving speed. Please help me to solve this.
I have write my code below :
public static string tempFile = @"D:\LiveData.txt";
private void websocket_MessageReceived(object sender, MessageReceivedEventArgs e)
{
using (w = System.IO.File.AppendText(tempFile))
{
Log(e.Message, w);
}
using (System.IO.StreamReader r = System.IO.File.OpenText(tempFile))
{
DumpLog(r);
}
}
public static void Log(string responseMessage, System.IO.TextWriter w)
{
w.WriteLine(responseMessage);
}
static string tempPath = @"D:\LongTimeHoldingDataFile.txt";
public static void DumpLog(System.IO.StreamReader r)
{
string line;
while ((line = r.ReadLine()) != null)
{
using (System.IO.StreamWriter w = System.IO.File.AppendText(tempPath))
{
w.WriteLine(line);
w.Close();
something();
}
}
r.Close();
var lines = System.IO.File.ReadAllLines(tempFile).Skip(1);
System.IO.File.WriteAllLines(tempFile, lines);
}
private const string DirPath = @"D:\seprateSymbolPair";
private const string Separator = @",";
public static void something()
{
System.IO.File.ReadLines(tempFile).ToList()
.AsParallel()
.Select(Newtonsoft.Json.JsonConvert.DeserializeObject<List<CryptoObject>>)
.ForAll(WriteRecord);
}
public static void WriteRecord(List<CryptoObject> data)
{
foreach (var item in data)
{
var fileNames = System.IO.Directory.GetFiles(DirPath, item.pair + ".txt", System.IO.SearchOption.AllDirectories);
foreach (var fileName in fileNames)
{
List<string> teFil = new List<string>();
try
{
var fileLines = System.IO.File.ReadAllLines(fileName).Skip(1).ToList();
teFil = fileLines;
writeLineTo(fileName, fileLines, item.pair, item.ap, (int)item.bp);
}
catch (Exception ex)
{
if (ex.Message.StartsWith("The process cannot access the file"))
{
System.Threading.Thread.Sleep(500);
writeLineTo(fileName, teFil, item.pair, item.ap, (int)item.bp);
}
}
}
}
}
public static void writeLineTo(string fileName, List<string> fileLines, string sym, double clo, int vol)
{
try
{
fileLines.Add(new StringBuilder().Append(sym).Append(Separator)
.Append(clo).Append(Separator).Append(vol).Append(Environment.NewLine)
.ToString());
System.IO.File.WriteAllLines(fileName, fileLines);
}
catch (Exception ex)
{
if (ex.Message.StartsWith("The process cannot access the file"))
{
System.Threading.Thread.Sleep(5);
fileLines.Add(new StringBuilder().Append(sym).Append(Separator)
.Append(clo).Append(Separator).Append(vol)
.Append(Environment.NewLine).ToString());
System.IO.File.WriteAllLines(fileName, fileLines);
}
}
}