From 84914923be8d2345b9db12858751dc7b81b4715e Mon Sep 17 00:00:00 2001
From: Aaron Cuevas Lopez <aacuelo@teleco.upv.es>
Date: Wed, 13 Apr 2016 04:05:28 +0200
Subject: [PATCH] Save timestamps in kwd files (WIP)
---
.../RecordEngine/HDF5FileFormat.cpp | 23 ++++-
.../KWIKFormat/RecordEngine/HDF5FileFormat.h | 2 +
.../KWIKFormat/RecordEngine/HDF5Recording.cpp | 88 +++++++++++++------
.../KWIKFormat/RecordEngine/HDF5Recording.h | 6 +-
.../RecordNode/OriginalRecording.cpp | 2 +-
.../Processors/RecordNode/OriginalRecording.h | 2 +-
Source/Processors/RecordNode/RecordEngine.h | 2 +-
7 files changed, 95 insertions(+), 30 deletions(-)
diff --git a/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp b/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp
index a5de05f57..e2c449ba0 100644
--- a/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp
+++ b/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp
@@ -40,6 +40,10 @@
#define SPIKE_CHUNK_YSIZE 40
#endif
+#ifndef TIMESTAMP_CHUNK_SIZE
+#define TIMESTAMP_CHUNK_SIZE 4
+#endif
+
#define MAX_TRANSFORM_SIZE 512
#define MAX_STR_SIZE 256
@@ -675,13 +679,21 @@ void KWDFile::startNewRecording(int recordingNumber, int nChannels, HDF5Recordin
CHECK_ERROR(createGroup(recordPath+"/application_data"));
// CHECK_ERROR(setAttributeArray(F32,info->bitVolts.getRawDataPointer(),info->bitVolts.size(),recordPath+"/application_data",String("channel_bit_volts")));
bitVoltsSet = createDataSet(F32, info->bitVolts.size(), 0, recordPath + "/application_data/channel_bit_volts");
- bitVoltsSet->writeDataBlock(info->bitVolts.size(), F32, info->bitVolts.getRawDataPointer());
+ if (bitVoltsSet.get())
+ bitVoltsSet->writeDataBlock(info->bitVolts.size(), F32, info->bitVolts.getRawDataPointer());
+ else
+ std::cerr << "Error creating bitvolts data set" << std::endl;
CHECK_ERROR(setAttribute(U8,&mSample,recordPath+"/application_data",String("is_multiSampleRate_data")));
CHECK_ERROR(setAttributeArray(F32,info->channelSampleRates.getRawDataPointer(),info->channelSampleRates.size(),recordPath+"/application_data",String("channel_sample_rates")));
recdata = createDataSet(I16,0,nChannels,CHUNK_XSIZE,recordPath+"/data");
if (!recdata.get())
std::cerr << "Error creating data set" << std::endl;
+
+ tsData = createDataSet(I64, 0, nChannels, TIMESTAMP_CHUNK_SIZE, recordPath + "/application_data/timestamps");
+ if (!tsData.get())
+ std::cerr << "Error creating timestamps data set" << std::endl;
+
curChan = nChannels;
}
@@ -694,6 +706,7 @@ void KWDFile::stopRecording()
CHECK_ERROR(setAttributeArray(U32,samples.getRawDataPointer(),samples.size(),path,"valid_samples"));
//ScopedPointer does the deletion and destructors the closings
recdata = nullptr;
+ tsData = nullptr;
}
int KWDFile::createFileStructure()
@@ -728,6 +741,14 @@ void KWDFile::writeRowData(int16* data, int nSamples, int channel)
}
}
+void KWDFile::writeTimestamps(int64* ts, int nTs, int channel)
+{
+ if (channel >= 0 && channel < nChannels)
+ {
+ CHECK_ERROR(tsData->writeDataRow(channel, nTs, I64, ts));
+ }
+}
+
//KWE File
KWEFile::KWEFile(String basename) : HDF5FileBase()
diff --git a/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.h b/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.h
index ca0f190f3..f98748373 100644
--- a/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.h
+++ b/Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.h
@@ -128,6 +128,7 @@ public:
void writeBlockData(int16* data, int nSamples);
void writeRowData(int16* data, int nSamples);
void writeRowData(int16* data, int nSamples, int channel);
+ void writeTimestamps(int64* ts, int nTs, int channel);
String getFileName();
protected:
@@ -140,6 +141,7 @@ private:
String filename;
bool multiSample;
ScopedPointer<HDF5RecordingData> recdata;
+ ScopedPointer<HDF5RecordingData> tsData;
JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(KWDFile);
};
diff --git a/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp b/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp
index 3806adccc..c5d545c4e 100644
--- a/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp
+++ b/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp
@@ -23,6 +23,8 @@
#include "HDF5Recording.h"
#define MAX_BUFFER_SIZE 10000
+#define CHANNEL_TIMESTAMP_PREALLOC_SIZE 16
+#define TIMESTAMP_EACH_NSAMPLES 1024
HDF5Recording::HDF5Recording() : processorIndex(-1), hasAcquired(false)
{
@@ -37,7 +39,7 @@ HDF5Recording::~HDF5Recording()
delete intBuffer;
}
-String HDF5Recording::getEngineID()
+String HDF5Recording::getEngineID() const
{
return "KWIK";
}
@@ -73,6 +75,8 @@ void HDF5Recording::resetChannels()
processorMap.clear();
infoArray.clear();
recordedChanToKWDChan.clear();
+ channelLeftOverSamples.clear();
+ channelTimestampArray.clear();
if (spikesFile)
spikesFile->resetChannels();
}
@@ -129,31 +133,11 @@ void HDF5Recording::openFiles(File rootFolder, int experimentNumber, int recordi
int procPos = processorRecPos[index];
recordedChanToKWDChan.add(procPos);
processorRecPos.set(index, procPos+1);
+ channelTimestampArray.add(new Array<int64>);
+ channelTimestampArray.getLast()->ensureStorageAllocated(CHANNEL_TIMESTAMP_PREALLOC_SIZE);
+ channelLeftOverSamples.add(0);
}
-#if 0
- for (int i = 0; i < processorMap.size(); i++)
- {
- int index = processorMap[i];
- if (getChannel(i)->getRecordState())
- {
- if (!fileArray[index]->isOpen())
- {
- fileArray[index]->initFile(getChannel(i)->nodeId,basepath);
- if (hasAcquired)
- infoArray[index]->start_time = (*timestamps)[getChannel(i)->sourceNodeId]; //the timestamps of the first channel
- else
- infoArray[index]->start_time = 0;
- }
- channelsPerProcessor.set(index, channelsPerProcessor[index] + 1);
- bitVoltsArray[index]->add(getChannel(i)->bitVolts);
- sampleRatesArray[index]->add(getChannel(i)->sampleRate);
- if (getChannel(i)->sampleRate != infoArray[index]->sample_rate)
- {
- infoArray[index]->multiSample = true;
- }
- }
- }
-#endif
+
for (int i = 0; i < fileArray.size(); i++)
{
if ((!fileArray[i]->isOpen()) && (fileArray[i]->isReadyToOpen()))
@@ -196,6 +180,18 @@ void HDF5Recording::closeFiles()
}
channelsPerProcessor.set(i, 0);
}
+ recordedChanToKWDChan.clear();
+ channelTimestampArray.clear();
+ channelLeftOverSamples.clear();
+}
+
+void HDF5Recording::startChannelBlock()
+{
+ int nCh = channelTimestampArray.size();
+ for (int i = 0; i < nCh; ++i)
+ {
+ channelTimestampArray[i]->clearQuick();
+ }
}
void HDF5Recording::writeData(int writeChannel, int realChannel, const float* buffer, int size)
@@ -208,6 +204,48 @@ void HDF5Recording::writeData(int writeChannel, int realChannel, const float* bu
fileArray[index]->writeRowData(intBuffer, size, recordedChanToKWDChan[writeChannel]);
// int64 t2 = Time::getHighResolutionTicks();
// std::cout << "record time: " << float(t2 - t1) / float(Time::getHighResolutionTicksPerSecond()) << std::endl;
+ int64 sampleOffset = channelLeftOverSamples[writeChannel];
+ if (writeChannel == 0)
+ std::cout << "Write " << size << " off " << sampleOffset << " ts " << getTimestamp(realChannel) << " - ";
+ if (sampleOffset + size >= TIMESTAMP_EACH_NSAMPLES)
+ {
+ int64 currentTimestamp = getTimestamp(realChannel);
+ if (sampleOffset > 0)
+ {
+ currentTimestamp = getTimestamp(realChannel) + TIMESTAMP_EACH_NSAMPLES - sampleOffset;
+ }
+
+ for (int samp = 0; samp < size; samp += TIMESTAMP_EACH_NSAMPLES)
+ {
+ if (writeChannel == 0)
+ std::cout << "w: " << currentTimestamp << " ";
+ channelTimestampArray[writeChannel]->add(currentTimestamp);
+ currentTimestamp += TIMESTAMP_EACH_NSAMPLES;
+ }
+
+ channelLeftOverSamples.set(writeChannel, (size + sampleOffset) % TIMESTAMP_EACH_NSAMPLES);
+ }
+ else
+ {
+ channelLeftOverSamples.set(writeChannel, sampleOffset + size);
+ }
+ if (writeChannel == 0)
+ std::cout << std::endl;
+}
+
+void HDF5Recording::endChannelBlock()
+{
+ int nCh = channelTimestampArray.size();
+ for (int ch = 0; ch < nCh; ++ch)
+ {
+ int tsSize = channelTimestampArray[ch]->size();
+ if (tsSize > 0)
+ {
+ int realChan = getRealChannel(ch);
+ int index = processorMap[getChannel(realChan)->recordIndex];
+ fileArray[index]->writeTimestamps(channelTimestampArray[ch]->getRawDataPointer(), tsSize, ch);
+ }
+ }
}
void HDF5Recording::writeEvent(int eventType, const MidiMessage& event, int64 timestamp)
diff --git a/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h b/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h
index a40f5f1e9..cc0ac27b9 100644
--- a/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h
+++ b/Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h
@@ -32,7 +32,7 @@ class HDF5Recording : public RecordEngine
public:
HDF5Recording();
~HDF5Recording();
- String getEngineID();
+ String getEngineID() const override;
void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
void closeFiles() override;
void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;
@@ -43,6 +43,8 @@ public:
void registerProcessor(const GenericProcessor* processor) override;
void resetChannels() override;
void startAcquisition() override;
+ void startChannelBlock() override;
+ void endChannelBlock() override;
static RecordEngineManager* getEngineManager();
private:
@@ -54,6 +56,8 @@ private:
Array<int> recordedChanToKWDChan;
OwnedArray<Array<float>> bitVoltsArray;
OwnedArray<Array<float>> sampleRatesArray;
+ OwnedArray<Array<int64>> channelTimestampArray;
+ Array<int> channelLeftOverSamples;
OwnedArray<KWDFile> fileArray;
OwnedArray<HDF5RecordingInfo> infoArray;
ScopedPointer<KWEFile> eventFile;
diff --git a/Source/Processors/RecordNode/OriginalRecording.cpp b/Source/Processors/RecordNode/OriginalRecording.cpp
index 5205d144e..7e5aab4d8 100644
--- a/Source/Processors/RecordNode/OriginalRecording.cpp
+++ b/Source/Processors/RecordNode/OriginalRecording.cpp
@@ -62,7 +62,7 @@ OriginalRecording::~OriginalRecording()
delete recordMarker;*/
}
-String OriginalRecording::getEngineID()
+String OriginalRecording::getEngineID() const
{
return "OPENEPHYS";
}
diff --git a/Source/Processors/RecordNode/OriginalRecording.h b/Source/Processors/RecordNode/OriginalRecording.h
index 8a7e7dd3f..0aa252e12 100644
--- a/Source/Processors/RecordNode/OriginalRecording.h
+++ b/Source/Processors/RecordNode/OriginalRecording.h
@@ -45,7 +45,7 @@ public:
~OriginalRecording();
void setParameter(EngineParameter& parameter);
- String getEngineID();
+ String getEngineID() const override;
void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
void closeFiles() override;
void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;
diff --git a/Source/Processors/RecordNode/RecordEngine.h b/Source/Processors/RecordNode/RecordEngine.h
index 1be3109c1..a22271351 100644
--- a/Source/Processors/RecordNode/RecordEngine.h
+++ b/Source/Processors/RecordNode/RecordEngine.h
@@ -59,7 +59,7 @@ class PLUGIN_API RecordEngine
public:
RecordEngine();
virtual ~RecordEngine();
- virtual String getEngineID() =0;
+ virtual String getEngineID() const =0;
/** All the public methods (except registerManager) are called by RecordNode or RecordingThread:
When acquisition starts (in the specified order):
--
GitLab