Skip to content
Snippets Groups Projects
Commit d224b0e0 authored by Aaron Cuevas Lopez's avatar Aaron Cuevas Lopez
Browse files

Modify NWB format so it can save all the data from the new event system

parent 0ce3effa
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Builds/VisualStudio2013/Plugins/Plugins.sln
+ 0
− 14
View file @ d224b0e0
......@@ -40,8 +40,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SpikeSorter", "SpikeSorter\
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ExampleProcessor", "ExampleProcessor\ExampleProcessor.vcxproj", "{767D282E-0BE5-4B35-874A-3B1ED925F06B}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PCIeRhythm", "PCIeRhythm\PCIeRhythm.vcxproj", "{8F019559-89D4-4C33-BA0F-FF330C57BA2B}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BinaryWriter", "BinaryWriter\BinaryWriter.vcxproj", "{9DB31964-F7E8-49B0-92E9-BAB2C35AF4B5}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LFP_Viewer_Beta", "LFP_Viewer_Beta\LFP_Viewer_Beta.vcxproj", "{A6E2C4F0-63A3-496B-8929-1B2785FDBBFD}"
......@@ -261,18 +259,6 @@ Global
{767D282E-0BE5-4B35-874A-3B1ED925F06B}.Release|Mixed Platforms.ActiveCfg = Release|x64
{767D282E-0BE5-4B35-874A-3B1ED925F06B}.Release|Win32.ActiveCfg = Release|Win32
{767D282E-0BE5-4B35-874A-3B1ED925F06B}.Release|x64.ActiveCfg = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|Mixed Platforms.ActiveCfg = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|Mixed Platforms.Build.0 = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|Win32.ActiveCfg = Debug|Win32
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|Win32.Build.0 = Debug|Win32
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|x64.ActiveCfg = Debug|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Debug|x64.Build.0 = Debug|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|Mixed Platforms.ActiveCfg = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|Mixed Platforms.Build.0 = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|Win32.ActiveCfg = Release|Win32
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|Win32.Build.0 = Release|Win32
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|x64.ActiveCfg = Release|x64
{8F019559-89D4-4C33-BA0F-FF330C57BA2B}.Release|x64.Build.0 = Release|x64
{9DB31964-F7E8-49B0-92E9-BAB2C35AF4B5}.Debug|Mixed Platforms.ActiveCfg = Release|x64
{9DB31964-F7E8-49B0-92E9-BAB2C35AF4B5}.Debug|Mixed Platforms.Build.0 = Release|x64
{9DB31964-F7E8-49B0-92E9-BAB2C35AF4B5}.Debug|Win32.ActiveCfg = Debug|Win32
......
Source/Plugins/NWBFormat/NWBFormat.cpp
+ 462
− 163
View file @ d224b0e0
This diff is collapsed.
Source/Plugins/NWBFormat/NWBFormat.h
+ 38
− 37
View file @ d224b0e0
......@@ -25,22 +25,22 @@
#define NWBFORMAT_H
#include <OpenEphysHDF5Lib/HDF5FileFormat.h>
#include <RecordingLib.h>
using namespace OpenEphysHDF5;
namespace NWBRecording
{
struct NWBRecordingInfo
typedef Array<const DataChannel*> ContinuousGroup;
class TimeSeries
{
String sourceName;
float bitVolts;
int processorId;
int sourceId;
int sourceSubIdx;
int nChannels;
int nSamplesPerSpike;
float sampleRate;
String spikeElectrodeName;
public:
ScopedPointer<HDF5RecordingData> baseDataSet;
ScopedPointer<HDF5RecordingData> timestampDataSet;
ScopedPointer<HDF5RecordingData> controlDataSet; //for all but spikes
ScopedPointer<HDF5RecordingData> ttlWordDataSet; //just for ttl events
OwnedArray<HDF5RecordingData> metaDataSet;
String basePath;
uint64 numSamples{ 0 };
};
class NWBFile : public HDF5FileBase
......@@ -48,13 +48,13 @@ namespace NWBRecording
public:
NWBFile(String fName, String ver, String idText); //with whatever arguments it's necessary
~NWBFile();
bool startNewRecording(int recordingNumber, const Array<NWBRecordingInfo>& continuousArray, const Array<NWBRecordingInfo>& electrodeArray);
bool startNewRecording(int recordingNumber, const Array<ContinuousGroup>& continuousArray,
const Array<const EventChannel*>& eventArray, const Array<const SpikeChannel*>& electrodeArray);
void stopRecording();
void writeData(int datasetID, int channel, int nSamples, const int16* data);
void writeData(int datasetID, int channel, int nSamples, const float* data, float bitVolts);
void writeTimestamps(int datasetID, int nSamples, const double* data);
void writeSpike(int electrodeId, const int16* data, double timestampSec);
void writeTTLEvent(int channel, int id, uint8 source, double timestampSec);
void writeMessage(const char* msg, double timestampSec);
void writeSpike(int electrodeId, const SpikeChannel* channel, const SpikeEvent* event);
void writeEvent(int eventID, const EventChannel* channel, const Event* event);
String getFileName() override;
void setXmlText(const String& xmlText);
......@@ -62,36 +62,37 @@ namespace NWBRecording
int createFileStructure() override;
private:
HDF5RecordingData* createRecordingStructures(String basePath, const NWBRecordingInfo& info, String helpText, int chunk_size, String ancestry);
void createTextDataSet(String path, String name, String text);
void createBinaryDataSet(String path, String name, HDF5FileBase::BaseDataType type, int length, void* data);
static HDF5FileBase::BaseDataType getEventH5Type(EventChannel::EventChannelTypes type, int length = 1);
static HDF5FileBase::BaseDataType getMetaDataH5Type(MetaDataDescriptor::MetaDataTypes type, int length = 1);
bool createTimeSeriesBase(String basePath, String source, String helpText, String description, StringArray ancestry);
bool createExtraInfo(String basePath, String name, String desc, String id, uint16 index, uint16 typeIndex);
HDF5RecordingData* createTimestampDataSet(String basePath, int chunk_size);
void createDataAttributes(String basePath, float conversion, float resolution, String unit);
bool createChannelMetaDataSets(String basePath, const MetaDataInfoObject* info);
bool createEventMetaDataSets(String basePath, TimeSeries* timeSeries, const MetaDataEventObject* info);
void writeEventMetaData(TimeSeries* timeSeries, const MetaDataEventObject* info, const MetaDataEvent* event);
const String filename;
const String GUIVersion;
OwnedArray<HDF5RecordingData> continuousDataSets;
OwnedArray<HDF5RecordingData> continuousDataSetsTS;
StringArray continuousBasePaths;
Array<uint64> numContinuousSamples;
Array<NWBRecordingInfo> continuousInfoStructs;
OwnedArray<HDF5RecordingData> spikeDataSets;
OwnedArray<HDF5RecordingData> spikeDataSetsTS;
StringArray spikeBasePaths;
Array<uint64> numSpikes;
Array<NWBRecordingInfo> spikeInfoStructs;
ScopedPointer<HDF5RecordingData> eventsDataSet;
ScopedPointer<HDF5RecordingData> eventsDataSetTS;
String eventsBasePath;
uint64 numEvents;
ScopedPointer<HDF5RecordingData> messagesDataSet;
ScopedPointer<HDF5RecordingData> messagesDataSetTS;
String messagesBasePath;
uint64 numMessages;
ScopedPointer<HDF5RecordingData> eventsControlDataSet;
OwnedArray<TimeSeries> continuousDataSets;
OwnedArray<TimeSeries> spikeDataSets;
OwnedArray<TimeSeries> eventDataSets;
const String* xmlText;
const String identifierText;
HeapBlock<float> scaledBuffer;
HeapBlock<int16> intBuffer;
size_t bufferSize;
JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(NWBFile);
};
......
Source/Plugins/NWBFormat/NWBRecording.cpp
+ 31
− 76
View file @ d224b0e0
......@@ -26,10 +26,9 @@
using namespace NWBRecording;
NWBRecordEngine::NWBRecordEngine() : bufferSize(MAX_BUFFER_SIZE)
NWBRecordEngine::NWBRecordEngine()
{
scaledBuffer.malloc(MAX_BUFFER_SIZE);
intBuffer.malloc(MAX_BUFFER_SIZE);
tsBuffer.malloc(MAX_BUFFER_SIZE);
}
......@@ -55,7 +54,7 @@
datasetIndexes.insertMultiple(0, 0, getNumRecordedChannels());
writeChannelIndexes.insertMultiple(0, 0, getNumRecordedChannels());
//Generate the continuous datasets info array, seeking for different combinations of recorded processor and source processor
int lastId = 0;
for (int proc = 0; proc < recProcs; proc++)
......@@ -69,15 +68,15 @@
const DataChannel* channelInfo = getDataChannel(realChan);
int sourceId = channelInfo->getSourceNodeID();
int sourceSubIdx = channelInfo->getSubProcessorIdx();
int nInfoArrays = continuousInfo.size();
int nInfoArrays = continuousChannels.size();
bool found = false;
for (int i = lastId; i < nInfoArrays; i++)
{
if (sourceId == continuousInfo[i].sourceId && sourceSubIdx == continuousInfo[i].sourceSubIdx)
if (sourceId == continuousChannels[i][0]->getSourceNodeID() && sourceSubIdx == continuousChannels[i][0]->getSubProcessorIdx())
{
//A dataset for the current processor from the current source is already present
writeChannelIndexes.set(recordedChan, continuousInfo[i].nChannels);
continuousInfo.getReference(i).nChannels += 1;
writeChannelIndexes.set(recordedChan, continuousChannels[i].size());
continuousChannels[i].add(getDataChannel(realChan));
datasetIndexes.set(recordedChan, i);
found = true;
break;
......@@ -85,31 +84,25 @@
}
if (!found) //a new dataset must be created
{
NWBRecordingInfo recInfo;
recInfo.bitVolts = channelInfo->getBitVolts();
recInfo.nChannels = 1;
recInfo.processorId = procInfo.processorId;
recInfo.sampleRate = channelInfo->getSampleRate();
recInfo.sourceName = "processor: " + channelInfo->getSourceName() + " (" + String(sourceId) + "." + String(sourceSubIdx) + ")";
recInfo.sourceId = sourceId;
recInfo.sourceSubIdx = sourceSubIdx;
recInfo.spikeElectrodeName = " ";
recInfo.nSamplesPerSpike = 0;
continuousInfo.add(recInfo);
ContinuousGroup newGroup;
newGroup.add(getDataChannel(realChan));
continuousChannels.add(newGroup);
datasetIndexes.set(recordedChan, nInfoArrays);
writeChannelIndexes.set(recordedChan, 0);
}
}
lastId = continuousInfo.size();
lastId = continuousChannels.size();
}
int nEvents = getNumRecordedEvents();
for (int i = 0; i < nEvents; i++)
eventChannels.add(getEventChannel(i));
//open the file
recordFile->open(getNumRecordedChannels() + continuousInfo.size()); //total channels + timestamp arrays, to create a big enough buffer
recordFile->open(getNumRecordedChannels() + continuousChannels.size() + eventChannels.size() + spikeChannels.size()); //total channels + timestamp arrays, to create a big enough buffer
//create the recording
recordFile->startNewRecording(recordingNumber, continuousInfo, spikeInfo);
recordFile->startNewRecording(recordingNumber, continuousChannels, eventChannels, spikeChannels);
}
......@@ -126,38 +119,24 @@
{
resetChannels(true);
}
void NWBRecordEngine::resetChannels(bool resetSpikes)
{
//Called at various points, should reset everything.
if (resetSpikes) //only clear this at actual reset, not when closing files.
spikeInfo.clear();
continuousInfo.clear();
if (resetSpikes)
spikeChannels.clear();
eventChannels.clear();
continuousChannels.clear();
datasetIndexes.clear();
writeChannelIndexes.clear();
scaledBuffer.malloc(MAX_BUFFER_SIZE);
intBuffer.malloc(MAX_BUFFER_SIZE);
tsBuffer.malloc(MAX_BUFFER_SIZE);
bufferSize = MAX_BUFFER_SIZE;
}
void NWBRecordEngine::writeData(int writeChannel, int realChannel, const float* buffer, int size)
{
if (size > bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on file close and reset.
{
std::cerr << "Write buffer overrun, resizing to" << size << std::endl;
bufferSize = size;
scaledBuffer.malloc(size);
intBuffer.malloc(size);
tsBuffer.malloc(size);
}
double multFactor = 1 / (float(0x7fff) * getDataChannel(realChannel)->getBitVolts());
FloatVectorOperations::copyWithMultiply(scaledBuffer.getData(), buffer, multFactor, size);
AudioDataConverters::convertFloatToInt16LE(scaledBuffer.getData(), intBuffer.getData(), size);
recordFile->writeData(datasetIndexes[writeChannel], writeChannelIndexes[writeChannel], size, intBuffer.getData());
recordFile->writeData(datasetIndexes[writeChannel], writeChannelIndexes[writeChannel], size, buffer, getDataChannel(realChannel)->getBitVolts());
/* All channels in a dataset have the same number of samples and share timestamps. But since this method is called
asynchronously, the timestamps might not be in sync during acquisition, so we chose a channel and write the
......@@ -178,51 +157,27 @@
void NWBRecordEngine::writeEvent(int eventIndex, const MidiMessage& event)
{
if (Event::getEventType(event) == EventChannel::TTL)
{
TTLEventPtr ttl = TTLEvent::deserializeFromMessage(event, getEventChannel(eventIndex));
recordFile->writeTTLEvent(ttl->getChannel(), ttl->getState() ? 1 : 0, ttl->getSourceID() , double(ttl->getTimestamp()) / getEventChannel(eventIndex)->getSampleRate());
}
else if (Event::getEventType(event) == EventChannel::TEXT)
{
TextEventPtr text = TextEvent::deserializeFromMessage(event, getEventChannel(eventIndex));
recordFile->writeMessage(text->getText().toUTF8(), double(text->getTimestamp()) / getEventChannel(eventIndex)->getSampleRate());
}
const EventChannel* channel = getEventChannel(eventIndex);
EventPtr eventStruct = Event::deserializeFromMessage(event, channel);
recordFile->writeEvent(eventIndex, channel, eventStruct);
}
void NWBRecordEngine::writeTimestampSyncText(uint16 sourceID, uint16 sourceIdx, int64 timestamp, float sourceSampleRate, String text)
{
recordFile->writeMessage(text.toUTF8(), double(timestamp) / sourceSampleRate);
}
void NWBRecordEngine::addSpikeElectrode(int index,const SpikeChannel* elec)
{
//Called during chain update by a processor that records spikes. Allows the RecordEngine to gather data about the electrode, which will usually
//be used in openfiles to be sent to startNewRecording so the electrode info is stored into the file.
NWBRecordingInfo info;
info.bitVolts = elec->getChannelBitVolts(0);
info.nChannels = elec->getNumChannels();
info.nSamplesPerSpike = elec->getTotalSamples();
info.processorId = elec->getCurrentNodeID();
info.sourceId = elec->getSourceNodeID();
info.sourceSubIdx = elec->getSubProcessorIdx();
info.sampleRate = elec->getSampleRate();
info.sourceName = elec->getSourceName() + " (" + String(info.sourceId) + "." + String(info.sourceSubIdx) + ")";
info.spikeElectrodeName = elec->getName();
spikeInfo.add(info);
spikeChannels.add(elec);
}
void NWBRecordEngine::writeSpike(int electrodeIndex, const SpikeEvent* spike)
{
const SpikeChannel* channel = getSpikeChannel(electrodeIndex);
int totalSamples = channel->getTotalSamples() * channel->getNumChannels();
double timestamp = double(spike->getTimestamp()) / channel->getSampleRate();
double multFactor = 1 / (float(0x7fff) * channel->getChannelBitVolts(0));
FloatVectorOperations::copyWithMultiply(scaledBuffer.getData(), spike->getDataPointer(), multFactor, totalSamples);
AudioDataConverters::convertFloatToInt16LE(scaledBuffer.getData(), intBuffer.getData(), totalSamples);
recordFile->writeSpike(electrodeIndex, intBuffer.getData(), timestamp);
recordFile->writeSpike(electrodeIndex, channel, spike);
}
RecordEngineManager* NWBRecordEngine::getEngineManager()
......
Source/Plugins/NWBFormat/NWBRecording.h
+ 5
− 6
View file @ d224b0e0
......@@ -49,17 +49,16 @@
private:
void resetChannels(bool resetSpikes);
Array<NWBRecordingInfo> continuousInfo;
Array<NWBRecordingInfo> spikeInfo;
ScopedPointer<NWBFile> recordFile;
Array<int> datasetIndexes;
Array<int> writeChannelIndexes;
HeapBlock<float> scaledBuffer;
HeapBlock<int16> intBuffer;
Array<ContinuousGroup> continuousChannels;
Array<const EventChannel*> eventChannels;
Array<const SpikeChannel*> spikeChannels;
HeapBlock<double> tsBuffer;
int bufferSize;
size_t bufferSize;
String identifierText;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment