merge with upstream

Source/Plugins/BinaryWriter/BinaryRecording.cpp

@@ -346,9 +346,17 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
 		jsonFile->setProperty("channels", jsonSpikeChannels.getReference(i));
 	}
 
-	Array<NpyType> msgType;
-	msgType.add(NpyType("sync_text", BaseType::CHAR, 256));
-	m_syncTextFile = new NpyFile(basepath + "sync_text.npy", msgType);
+	File syncFile = File(basepath + "sync_messages.txt");
+	Result res = syncFile.create();
+	if (res.failed())
+	{
+		std::cerr << "Error creating sync text file:" << res.getErrorMessage() << std::endl;
+	}
+	else
+	{
+		m_syncTextFile = syncFile.createOutputStream();
+	}
+	
 	m_recordingNum = recordingNumber;
 
 	DynamicObject::Ptr jsonSettingsFile = new DynamicObject();
@@ -504,8 +512,8 @@ void BinaryRecording::writeData(int writeChannel, int realChannel, const float*
 	double multFactor = 1 / (float(0x7fff) * getDataChannel(realChannel)->getBitVolts());
 	FloatVectorOperations::copyWithMultiply(m_scaledBuffer.getData(), buffer, multFactor, size);
 	AudioDataConverters::convertFloatToInt16LE(m_scaledBuffer.getData(), m_intBuffer.getData(), size);
-
-	m_DataFiles[m_fileIndexes[writeChannel]]->writeChannel(getTimestamp(writeChannel)-m_startTS[writeChannel],m_channelIndexes[writeChannel],m_intBuffer.getData(),size);
+	int fileIndex = m_fileIndexes[writeChannel];
+	m_DataFiles[fileIndex]->writeChannel(getTimestamp(writeChannel) - m_startTS[writeChannel], m_channelIndexes[writeChannel], m_intBuffer.getData(), size);
 
 	if (m_channelIndexes[writeChannel] == 0)
 	{
@@ -515,7 +523,8 @@ void BinaryRecording::writeData(int writeChannel, int realChannel, const float*
 		{
 			m_tsBuffer[i] = (baseTS + i);
 		}
-		m_dataTimestampFiles[m_fileIndexes[writeChannel]]->writeData(m_tsBuffer, size*sizeof(int64));
+		m_dataTimestampFiles[fileIndex]->writeData(m_tsBuffer, size*sizeof(int64));
+		m_dataTimestampFiles[fileIndex]->increaseRecordCount(size);
 	}
 }
 
@@ -576,12 +585,14 @@ void BinaryRecording::writeEvent(int eventIndex, const MidiMessage& event)
 		chanFile->writeData(&chan, sizeof(uint16));
 	}
 	writeEventMetaData(ev.get(), rec->metaDataFile);
+	increaseEventCounts(rec);
 }
 
 void BinaryRecording::writeTimestampSyncText(uint16 sourceID, uint16 sourceIdx, int64 timestamp, float, String text)
 {
-	text.paddedRight(' ', 256);
-	m_syncTextFile->writeData(text.toUTF8(), 256);
+	if (!m_syncTextFile)
+		return;
+	m_syncTextFile->writeText(text + "\n", false, false);
 }
 
 
@@ -593,7 +604,7 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike)
 	uint16 spikeChannel = m_spikeChannelIndexes[electrodeIndex];
 
 	int totalSamples = channel->getTotalSamples() * channel->getNumChannels();
-	
+
 
 	if (totalSamples > m_bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on file close and reset.
 	{
@@ -629,8 +640,16 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike)
 
 	uint16 sortedID = spike->getSortedID();
 	sortedFile->writeData(&sortedID, sizeof(uint16));
+	increaseEventCounts(rec);
+}
 
-	writeEventMetaData(spike, rec->metaDataFile);
+void BinaryRecording::increaseEventCounts(EventRecording* rec)
+{
+	rec->mainFile->increaseRecordCount();
+	rec->timestampFile->increaseRecordCount();
+	if (rec->extraFile) rec->extraFile->increaseRecordCount();
+	if (rec->channelFile) rec->channelFile->increaseRecordCount();
+	if (rec->metaDataFile) rec->metaDataFile->increaseRecordCount();
 }
 
 RecordEngineManager* BinaryRecording::getEngineManager()

Source/Plugins/BinaryWriter/BinaryRecording.h

@@ -84,6 +84,7 @@ namespace BinaryRecordingEngine
 		NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject);
 		void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject);
 		void writeEventMetaData(const MetaDataEvent* event, NpyFile* file);
+		void increaseEventCounts(EventRecording* rec);
 		static String jsonTypeValue(BaseType type);
 
 		SpikeMode m_spikeMode;
@@ -102,7 +103,7 @@ namespace BinaryRecordingEngine
 		OwnedArray<EventRecording> m_eventFiles;
 		OwnedArray<EventRecording> m_spikeFiles;
 		OwnedArray<NpyFile> m_dataTimestampFiles;
-		ScopedPointer<NpyFile> m_syncTextFile;
+		ScopedPointer<FileOutputStream> m_syncTextFile;
 
 		Array<unsigned int> m_spikeFileIndexes;
 		Array<uint16> m_spikeChannelIndexes;

Source/Plugins/BinaryWriter/NpyFile.cpp

+/*
+------------------------------------------------------------------
+
+This file is part of the Open Ephys GUI
+Copyright (C) 2017 Open Ephys
+
+------------------------------------------------------------------
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "NpyFile.h"
+
+using namespace BinaryRecordingEngine;
+
+NpyFile::NpyFile(String path, const Array<NpyType>& typeList)
+{
+	File file(path);
+	Result res = file.create();
+	if (res.failed())
+	{
+		std::cerr << "Error creating file " << path << ":" << res.getErrorMessage() << std::endl;
+		return;
+	}
+	m_file = file.createOutputStream();
+	if (!m_file)
+		return;
+
+	m_okOpen = true;
+
+	String header = "{'descr': [";
+
+	int nTypes = typeList.size();
+	String name;
+
+	for (int i = 0; i < nTypes; i++)
+	{
+		NpyType& type = typeList.getReference(i);
+		if (i > 0) header += ", ";
+		header += "('" + type.getName() + "', '" + type.getTypeString() + "', (" + String(type.getTypeLength()) + ",))";
+	}
+	header += "], 'fortran_order': False, 'shape': (1,), }";
+
+	int headLength = header.length();
+	int padding = (int((headLength + 30 ) / 16) + 1) * 16;
+	header = header.paddedRight(' ', padding);
+	header += '\n';
+	
+	uint8 magicNum = 0x093;
+	m_file->write(&magicNum, sizeof(uint8));
+	String magic = "NUMPY";
+	uint16 len = header.length();
+	m_file->write(magic.toUTF8(), magic.getNumBytesAsUTF8());
+	uint16 ver = 0x0001;
+	m_file->write(&ver, sizeof(uint16));
+	m_file->write(&len, sizeof(uint16));
+	m_file->write(header.toUTF8(), len);
+	m_countPos = headLength + 4; //10-6
+	
+}
+
+NpyFile::~NpyFile()
+{
+	if (m_file->setPosition(m_countPos))
+	{
+		String newShape = String(m_recordCount) + ",), }";
+		m_file->write(newShape.toUTF8(), newShape.getNumBytesAsUTF8());
+	}
+	else
+	{
+		std::cerr << "Error. Unable to seek to update header on file " << m_file->getFile().getFullPathName() << std::endl;
+	}
+}
+
+void NpyFile::writeData(const void* data, size_t size)
+{
+	m_file->write(data, size);
+}
+
+void NpyFile::increaseRecordCount(int count)
+{
+	m_recordCount += count;
+}
+
+
+NpyType::NpyType(String n, BaseType t, size_t l)
+	: name(n), type(t), length(l)
+{
+}
+
+String NpyType::getTypeString() const
+{
+	switch (type)
+	{
+	case BaseType::CHAR:
+		return "S" + String(length + 1); //account for the null separator
+	case BaseType::INT8:
+		return "<i1";
+	case BaseType::UINT8:
+		return "<u1";
+	case BaseType::INT16:
+		return "<i2";
+	case BaseType::UINT16:
+		return "<u2";
+	case BaseType::INT32:
+		return "<i4";
+	case BaseType::UINT32:
+		return "<u4";
+	case BaseType::INT64:
+		return "<i8";
+	case BaseType::UINT64:
+		return "<u8";
+	case BaseType::FLOAT:
+		return "<f4";
+	case BaseType::DOUBLE:
+		return "<f8";
+	default:
+		return "<b1";
+	}
+}
+
+int NpyType::getTypeLength() const
+{
+	if (type == BaseType::CHAR)
+		return 1;
+	else
+		return length;
+}
+
+String NpyType::getName() const
+{
+	return name;
+}
\ No newline at end of file

Source/Plugins/BinaryWriter/NpyFile.h

+/*
+------------------------------------------------------------------
+
+This file is part of the Open Ephys GUI
+Copyright (C) 2017 Open Ephys
+
+------------------------------------------------------------------
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef NPYFILE_H
+#define NPYFILE_H
+
+#include <RecordingLib.h>
+
+namespace BinaryRecordingEngine
+{
+
+	class NpyType
+	{
+	public:
+		NpyType(String, BaseType, size_t);
+		String getName() const;
+		String getTypeString() const;
+		int getTypeLength() const;
+	private:
+		String name;
+		BaseType type;
+		size_t length;
+	};
+
+	class NpyFile
+	{
+	public:
+		NpyFile(String path, const Array<NpyType>& typeList);
+		~NpyFile();
+		void writeData(const void* data, size_t size);
+		void increaseRecordCount(int count = 1);
+	private:
+		ScopedPointer<FileOutputStream> m_file;
+		bool m_okOpen{ false };
+		int64 m_recordCount{ 0 };
+		size_t m_countPos;
+	};
+
+};
+#endif
\ No newline at end of file

Source/Processors/Channel/MetaData.cpp

@@ -71,7 +71,13 @@ MetaDataDescriptor& MetaDataDescriptor::operator=(const MetaDataDescriptor& othe
 
 MetaDataDescriptor::MetaDataTypes MetaDataDescriptor::getType() const { return m_type; }
 unsigned int MetaDataDescriptor::getLength() const { return m_length; }
-size_t MetaDataDescriptor::getDataSize() const { return m_length*getTypeSize(m_type); }
+size_t MetaDataDescriptor::getDataSize() const 
+{ 
+	if (m_type == CHAR)
+		return m_length*getTypeSize(m_type) + 1; //account for the null-rerminator
+	else
+		return m_length*getTypeSize(m_type);
+}
 String MetaDataDescriptor::getName() const { return m_name; }
 String MetaDataDescriptor::getDescription() const { return m_description; }
 String MetaDataDescriptor::getIdentifier() const { return m_identifier; }
@@ -120,14 +126,14 @@ size_t MetaDataDescriptor::getTypeSize(MetaDataDescriptor::MetaDataTypes type)
 
 //This would be so much easier if VS2012 supported delegating constructors...
 MetaDataValue::MetaDataValue(MetaDataDescriptor::MetaDataTypes t, unsigned int length, const void* d)
-	: m_type(t), m_length(length), m_size(length*MetaDataDescriptor::getTypeSize(t))
+	: m_type(t), m_length(length), m_size(getSize(t, length))
 {	
 	allocSpace();
 	setValue(d);
 }
 
 MetaDataValue::MetaDataValue(MetaDataDescriptor::MetaDataTypes t, unsigned int length) 
-	: m_type(t), m_length(length), m_size(length*MetaDataDescriptor::getTypeSize(t))
+	: m_type(t), m_length(length), m_size(getSize(t, length))
 {
 	allocSpace();
 }
@@ -175,6 +181,14 @@ void MetaDataValue::allocSpace()
 	m_data.calloc(m_size);
 }
 
+size_t MetaDataValue::getSize(MetaDataDescriptor::MetaDataTypes type, unsigned int length)
+{
+	if (type == MetaDataDescriptor::CHAR)
+		return length*MetaDataDescriptor::getTypeSize(type) + 1; //account for the null-rerminator
+	else
+		return length*MetaDataDescriptor::getTypeSize(type);
+}
+
 MetaDataValue::MetaDataValue(const MetaDataValue& v)
 	: ReferenceCountedObject(),
 	m_type(v.m_type), m_length(v.m_length), m_size(v.m_size)

Source/Processors/Channel/MetaData.h

@@ -162,6 +162,7 @@ private:
 	MetaDataValue() = delete;
 	void setValue(const void* data);
 	void allocSpace();
+	static size_t getSize(MetaDataDescriptor::MetaDataTypes type, unsigned int length);
 	HeapBlock<char> m_data;
 	MetaDataDescriptor::MetaDataTypes m_type;
 	unsigned int m_length;