diff --git a/Source/Plugins/BinaryWriter/BinaryRecording.cpp b/Source/Plugins/BinaryWriter/BinaryRecording.cpp index b13ee7123bd6925ccf8da8a50c6e527ac44d8de2..66217317a0f06b2c239d694c2b3ddcbf5f43354d 100644 --- a/Source/Plugins/BinaryWriter/BinaryRecording.cpp +++ b/Source/Plugins/BinaryWriter/BinaryRecording.cpp @@ -346,9 +346,17 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor jsonFile->setProperty("channels", jsonSpikeChannels.getReference(i)); } - Array<NpyType> msgType; - msgType.add(NpyType("sync_text", BaseType::CHAR, 256)); - m_syncTextFile = new NpyFile(basepath + "sync_text.npy", msgType); + File syncFile = File(basepath + "sync_messages.txt"); + Result res = syncFile.create(); + if (res.failed()) + { + std::cerr << "Error creating sync text file:" << res.getErrorMessage() << std::endl; + } + else + { + m_syncTextFile = syncFile.createOutputStream(); + } + m_recordingNum = recordingNumber; DynamicObject::Ptr jsonSettingsFile = new DynamicObject(); @@ -504,8 +512,8 @@ void BinaryRecording::writeData(int writeChannel, int realChannel, const float* double multFactor = 1 / (float(0x7fff) * getDataChannel(realChannel)->getBitVolts()); FloatVectorOperations::copyWithMultiply(m_scaledBuffer.getData(), buffer, multFactor, size); AudioDataConverters::convertFloatToInt16LE(m_scaledBuffer.getData(), m_intBuffer.getData(), size); - - m_DataFiles[m_fileIndexes[writeChannel]]->writeChannel(getTimestamp(writeChannel)-m_startTS[writeChannel],m_channelIndexes[writeChannel],m_intBuffer.getData(),size); + int fileIndex = m_fileIndexes[writeChannel]; + m_DataFiles[fileIndex]->writeChannel(getTimestamp(writeChannel) - m_startTS[writeChannel], m_channelIndexes[writeChannel], m_intBuffer.getData(), size); if (m_channelIndexes[writeChannel] == 0) { @@ -515,7 +523,8 @@ void BinaryRecording::writeData(int writeChannel, int realChannel, const float* { m_tsBuffer[i] = (baseTS + i); } - m_dataTimestampFiles[m_fileIndexes[writeChannel]]->writeData(m_tsBuffer, size*sizeof(int64)); + m_dataTimestampFiles[fileIndex]->writeData(m_tsBuffer, size*sizeof(int64)); + m_dataTimestampFiles[fileIndex]->increaseRecordCount(size); } } @@ -576,12 +585,14 @@ void BinaryRecording::writeEvent(int eventIndex, const MidiMessage& event) chanFile->writeData(&chan, sizeof(uint16)); } writeEventMetaData(ev.get(), rec->metaDataFile); + increaseEventCounts(rec); } void BinaryRecording::writeTimestampSyncText(uint16 sourceID, uint16 sourceIdx, int64 timestamp, float, String text) { - text.paddedRight(' ', 256); - m_syncTextFile->writeData(text.toUTF8(), 256); + if (!m_syncTextFile) + return; + m_syncTextFile->writeText(text + "\n", false, false); } @@ -593,7 +604,7 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) uint16 spikeChannel = m_spikeChannelIndexes[electrodeIndex]; int totalSamples = channel->getTotalSamples() * channel->getNumChannels(); - + if (totalSamples > m_bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on file close and reset. { @@ -629,8 +640,16 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) uint16 sortedID = spike->getSortedID(); sortedFile->writeData(&sortedID, sizeof(uint16)); + increaseEventCounts(rec); +} - writeEventMetaData(spike, rec->metaDataFile); +void BinaryRecording::increaseEventCounts(EventRecording* rec) +{ + rec->mainFile->increaseRecordCount(); + rec->timestampFile->increaseRecordCount(); + if (rec->extraFile) rec->extraFile->increaseRecordCount(); + if (rec->channelFile) rec->channelFile->increaseRecordCount(); + if (rec->metaDataFile) rec->metaDataFile->increaseRecordCount(); } RecordEngineManager* BinaryRecording::getEngineManager() diff --git a/Source/Plugins/BinaryWriter/BinaryRecording.h b/Source/Plugins/BinaryWriter/BinaryRecording.h index e0493fac8926f7fff26d2a5e95ecf37fe586fcef..962c5a7a89457033dc8fa525baf9fd498d73382b 100644 --- a/Source/Plugins/BinaryWriter/BinaryRecording.h +++ b/Source/Plugins/BinaryWriter/BinaryRecording.h @@ -84,6 +84,7 @@ namespace BinaryRecordingEngine NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject); void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject); void writeEventMetaData(const MetaDataEvent* event, NpyFile* file); + void increaseEventCounts(EventRecording* rec); static String jsonTypeValue(BaseType type); SpikeMode m_spikeMode; @@ -102,7 +103,7 @@ namespace BinaryRecordingEngine OwnedArray<EventRecording> m_eventFiles; OwnedArray<EventRecording> m_spikeFiles; OwnedArray<NpyFile> m_dataTimestampFiles; - ScopedPointer<NpyFile> m_syncTextFile; + ScopedPointer<FileOutputStream> m_syncTextFile; Array<unsigned int> m_spikeFileIndexes; Array<uint16> m_spikeChannelIndexes; diff --git a/Source/Plugins/BinaryWriter/NpyFile.cpp b/Source/Plugins/BinaryWriter/NpyFile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..60e822bad9ddcfca725e9b3d1849e316971c8c83 --- /dev/null +++ b/Source/Plugins/BinaryWriter/NpyFile.cpp @@ -0,0 +1,145 @@ +/* +------------------------------------------------------------------ + +This file is part of the Open Ephys GUI +Copyright (C) 2017 Open Ephys + +------------------------------------------------------------------ + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. + +*/ + +#include "NpyFile.h" + +using namespace BinaryRecordingEngine; + +NpyFile::NpyFile(String path, const Array<NpyType>& typeList) +{ + File file(path); + Result res = file.create(); + if (res.failed()) + { + std::cerr << "Error creating file " << path << ":" << res.getErrorMessage() << std::endl; + return; + } + m_file = file.createOutputStream(); + if (!m_file) + return; + + m_okOpen = true; + + String header = "{'descr': ["; + + int nTypes = typeList.size(); + String name; + + for (int i = 0; i < nTypes; i++) + { + NpyType& type = typeList.getReference(i); + if (i > 0) header += ", "; + header += "('" + type.getName() + "', '" + type.getTypeString() + "', (" + String(type.getTypeLength()) + ",))"; + } + header += "], 'fortran_order': False, 'shape': (1,), }"; + + int headLength = header.length(); + int padding = (int((headLength + 30 ) / 16) + 1) * 16; + header = header.paddedRight(' ', padding); + header += '\n'; + + uint8 magicNum = 0x093; + m_file->write(&magicNum, sizeof(uint8)); + String magic = "NUMPY"; + uint16 len = header.length(); + m_file->write(magic.toUTF8(), magic.getNumBytesAsUTF8()); + uint16 ver = 0x0001; + m_file->write(&ver, sizeof(uint16)); + m_file->write(&len, sizeof(uint16)); + m_file->write(header.toUTF8(), len); + m_countPos = headLength + 4; //10-6 + +} + +NpyFile::~NpyFile() +{ + if (m_file->setPosition(m_countPos)) + { + String newShape = String(m_recordCount) + ",), }"; + m_file->write(newShape.toUTF8(), newShape.getNumBytesAsUTF8()); + } + else + { + std::cerr << "Error. Unable to seek to update header on file " << m_file->getFile().getFullPathName() << std::endl; + } +} + +void NpyFile::writeData(const void* data, size_t size) +{ + m_file->write(data, size); +} + +void NpyFile::increaseRecordCount(int count) +{ + m_recordCount += count; +} + + +NpyType::NpyType(String n, BaseType t, size_t l) + : name(n), type(t), length(l) +{ +} + +String NpyType::getTypeString() const +{ + switch (type) + { + case BaseType::CHAR: + return "S" + String(length + 1); //account for the null separator + case BaseType::INT8: + return "<i1"; + case BaseType::UINT8: + return "<u1"; + case BaseType::INT16: + return "<i2"; + case BaseType::UINT16: + return "<u2"; + case BaseType::INT32: + return "<i4"; + case BaseType::UINT32: + return "<u4"; + case BaseType::INT64: + return "<i8"; + case BaseType::UINT64: + return "<u8"; + case BaseType::FLOAT: + return "<f4"; + case BaseType::DOUBLE: + return "<f8"; + default: + return "<b1"; + } +} + +int NpyType::getTypeLength() const +{ + if (type == BaseType::CHAR) + return 1; + else + return length; +} + +String NpyType::getName() const +{ + return name; +} \ No newline at end of file diff --git a/Source/Plugins/BinaryWriter/NpyFile.h b/Source/Plugins/BinaryWriter/NpyFile.h new file mode 100644 index 0000000000000000000000000000000000000000..a3bcda482fb66640e8ed951ab52d60db2a5cda71 --- /dev/null +++ b/Source/Plugins/BinaryWriter/NpyFile.h @@ -0,0 +1,60 @@ +/* +------------------------------------------------------------------ + +This file is part of the Open Ephys GUI +Copyright (C) 2017 Open Ephys + +------------------------------------------------------------------ + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. + +*/ + +#ifndef NPYFILE_H +#define NPYFILE_H + +#include <RecordingLib.h> + +namespace BinaryRecordingEngine +{ + + class NpyType + { + public: + NpyType(String, BaseType, size_t); + String getName() const; + String getTypeString() const; + int getTypeLength() const; + private: + String name; + BaseType type; + size_t length; + }; + + class NpyFile + { + public: + NpyFile(String path, const Array<NpyType>& typeList); + ~NpyFile(); + void writeData(const void* data, size_t size); + void increaseRecordCount(int count = 1); + private: + ScopedPointer<FileOutputStream> m_file; + bool m_okOpen{ false }; + int64 m_recordCount{ 0 }; + size_t m_countPos; + }; + +}; +#endif \ No newline at end of file diff --git a/Source/Processors/Channel/MetaData.cpp b/Source/Processors/Channel/MetaData.cpp index 8c6711b16d9c5775ed510c9e130fa29d3b436a46..da1ddbf2aa73348a301a9d4ce890d87393d003e7 100644 --- a/Source/Processors/Channel/MetaData.cpp +++ b/Source/Processors/Channel/MetaData.cpp @@ -71,7 +71,13 @@ MetaDataDescriptor& MetaDataDescriptor::operator=(const MetaDataDescriptor& othe MetaDataDescriptor::MetaDataTypes MetaDataDescriptor::getType() const { return m_type; } unsigned int MetaDataDescriptor::getLength() const { return m_length; } -size_t MetaDataDescriptor::getDataSize() const { return m_length*getTypeSize(m_type); } +size_t MetaDataDescriptor::getDataSize() const +{ + if (m_type == CHAR) + return m_length*getTypeSize(m_type) + 1; //account for the null-rerminator + else + return m_length*getTypeSize(m_type); +} String MetaDataDescriptor::getName() const { return m_name; } String MetaDataDescriptor::getDescription() const { return m_description; } String MetaDataDescriptor::getIdentifier() const { return m_identifier; } @@ -120,14 +126,14 @@ size_t MetaDataDescriptor::getTypeSize(MetaDataDescriptor::MetaDataTypes type) //This would be so much easier if VS2012 supported delegating constructors... MetaDataValue::MetaDataValue(MetaDataDescriptor::MetaDataTypes t, unsigned int length, const void* d) - : m_type(t), m_length(length), m_size(length*MetaDataDescriptor::getTypeSize(t)) + : m_type(t), m_length(length), m_size(getSize(t, length)) { allocSpace(); setValue(d); } MetaDataValue::MetaDataValue(MetaDataDescriptor::MetaDataTypes t, unsigned int length) - : m_type(t), m_length(length), m_size(length*MetaDataDescriptor::getTypeSize(t)) + : m_type(t), m_length(length), m_size(getSize(t, length)) { allocSpace(); } @@ -175,6 +181,14 @@ void MetaDataValue::allocSpace() m_data.calloc(m_size); } +size_t MetaDataValue::getSize(MetaDataDescriptor::MetaDataTypes type, unsigned int length) +{ + if (type == MetaDataDescriptor::CHAR) + return length*MetaDataDescriptor::getTypeSize(type) + 1; //account for the null-rerminator + else + return length*MetaDataDescriptor::getTypeSize(type); +} + MetaDataValue::MetaDataValue(const MetaDataValue& v) : ReferenceCountedObject(), m_type(v.m_type), m_length(v.m_length), m_size(v.m_size) diff --git a/Source/Processors/Channel/MetaData.h b/Source/Processors/Channel/MetaData.h index 88f316715f1a624a4b106f4d20dd78ed8c868486..0c36b2924c6a1e0757670fc51cb11bc854f178c0 100644 --- a/Source/Processors/Channel/MetaData.h +++ b/Source/Processors/Channel/MetaData.h @@ -162,6 +162,7 @@ private: MetaDataValue() = delete; void setValue(const void* data); void allocSpace(); + static size_t getSize(MetaDataDescriptor::MetaDataTypes type, unsigned int length); HeapBlock<char> m_data; MetaDataDescriptor::MetaDataTypes m_type; unsigned int m_length;