Time Major bitmode fix, Stokes decimation fix, Stokes Vector output mode

Third time's the charm? Revert and reapply changes meant to be in "Fix the skip_..."

I have no idea what the hell happened during this merge that so many little changes got reverted.

Time-major kernels were ignoring the output bitmode, which would cause issues when processing 16-bit data.
Stokes decimation was ignoring the 16x time scrunch, issue from the C++ upgrade (whoops.)
Add in mode 15[0..4] to process all stokes vectors at once.

Makefile

@@ -10,6 +10,7 @@ CXX		= g++
 endif
 
 LIB_VER = 0.3
+LIB_VER_MINOR = 4
 CLI_VER = 0.2
 
 # Detemrine the max threads per socket to speed up execution via OpenMP with ICC (GCC falls over if we set too many)
@@ -49,17 +50,17 @@ all: $(CLI_OBJECTS) library
 	$(CXX) $(CXXFLAGS) src/CLI/lofar_cli_guppi_raw.o $(CLI_META_OBJECTS) $(LIBRARY_TARGET) -o ./lofar_udp_guppi_raw $(LFLAGS)
 
 library: $(OBJECTS)
-	$(AR) rc $(LIBRARY_TARGET).$(LIB_VER) $(OBJECTS)
-	cp ./$(LIBRARY_TARGET).$(LIB_VER) ./$(LIBRARY_TARGET) 
+	$(AR) rc $(LIBRARY_TARGET).$(LIB_VER).$(LIB_VER_MINOR) $(OBJECTS)
+	ln -s ./$(LIBRARY_TARGET).$(LIB_VER).$(LIB_VER_MINOR) ./$(LIBRARY_TARGET) 
 
 install: all
 	mkdir -p $(PREFIX)/bin/ && mkdir -p $(PREFIX)/include/
 	cp ./lofar_udp_extractor $(PREFIX)/bin/
 	cp ./lofar_udp_guppi_raw $(PREFIX)/bin/
 	cp ./src/lib/*.h $(PREFIX)/include/
 	cp ./src/lib/*.hpp $(PREFIX)/include/
-	cp ./*.a* ${PREFIX}/lib/
-	cp ./*.a ${PREFIX}/lib/	
+	cp -P ./*.a* ${PREFIX}/lib/
+	cp -P ./*.a ${PREFIX}/lib/	
 	cp ./mockHeader/mockHeader $(PREFIX)/bin/; exit 0;
 
 install-local: all
@@ -68,8 +69,8 @@ install-local: all
 	cp ./lofar_udp_guppi_raw ~/.local/bin/
 	cp ./src/lib/*.h ~/.local/include/
 	cp ./src/lib/*.hpp ~/.local/include/
-	cp ./*.a* ~/.local/lib/
-	cp ./*.a ~/.local/lib/
+	cp -P ./*.a* ~/.local/lib/
+	cp -P ./*.a ~/.local/lib/
 	cp ./mockHeader/mockHeader ~/.local/bin/; exit 0;
 
 clean:
@@ -81,6 +82,23 @@ clean:
 	rm ./lofar_udp_extractor; exit 0;
 	rm ./lofar_udp_guppi_raw; exit 0;
 
+remove:
+	rm $(PREFIX)/bin/lofar_udp_extractor
+	rm $(PREFIX)/bin/lofar_udp_guppi_raw
+	cd src/lib/; find . -name "*.hpp" -exec rm $(PREFIX)/include/{} \;
+	cd src/lib/; find . -name "*.h" -exec rm $(PREFIX)/include/{} \;
+	find . -name "*.a" -exec rm $(PREFIX)/lib/{} \;
+	find . -name "*.a.*" -exec rm $(PREFIX)/lib/{} \;
+	make clean
+
+remove-local:
+	rm ~/.local/bin/lofar_udp_extractor
+	rm ~/.local/bin/lofar_udp_guppi_raw
+	cd src/lib/; find . -name "*.hpp" -exec rm ~/.local/include/{} \;
+	cd src/lib/; find . -name "*.h" -exec rm ~/.local/include/{} \;
+	find . -name "*.a" -exec rm ~/.local/lib/{} \;
+	find . -name "*.a.*" -exec rm ~/.local/lib/{} \;
+	make clean
 
 
 

README.md

@@ -2,7 +2,7 @@ udpPacketManager
 ================
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4043970.svg)](https://doi.org/10.5281/zenodo.4043970)
 
-udpPacketManager is a C library developed to handle reading and processing packet streams from international LOFAR stations. It is used at the Irish LOFAR station (I-LOFAR) in conjunction with Olaf Wucknitz's (MPIfRA) VLBI recording software, but in principle can be used with any packet capture that keep the last 16 bytes of the UDP header attached to CEP packets.
+udpPacketManager is a C library developed to handle reading and processing packet streams from international LOFAR stations. It is used at the Irish LOFAR station (I-LOFAR) in conjunction with Olaf Wucknitz's (MPIfRA) VLBI recording software, but in principle can be used with any packet capture that keeps the last 16 bytes of the UDP header attached to CEP packets.
 
 This library allows for the entire or partial extraction and processing of LOFAR CEP packet streams, re-aligning data to account for packet loss or misalignment on the first packet, to produce one of several data products, ranging from raw voltages (reordered or not) to stokes vector outputs.
 

docs/README_CLI.md

@@ -141,14 +141,18 @@ By default, we define a number of base Stokes parameter outputs each at a multip
 - Take the input data, apply (20) and then combine the polarizations to form a 32-bit floating point Stokes Q for each frequency sample
 - N input files -> 1 output file
 
-#### 110: "Stokes U"
+#### 120: "Stokes U"
 - Take the input data, apply (20) and then combine the polarizations to form a 32-bit floating point Stokes U for each frequency sample
 - N input files -> 1 output file
 
-#### 110: "Stokes V"
+#### 130: "Stokes V"
 - Take the input data, apply (20) and then combine the polarizations to form a 32-bit floating point Stokes V for each frequency sample
 - N input files -> 1 output file
 
+#### 150: "Stokes Vector"
+- Take the input data, apply (20), and then combine the polariszation to form 4 output 32-bit floating poit Stoke s(I, Q, U, V) filterbanks for each frequency sample
+- N input files -> 4 output files
+
 #### Time decimation
 We also offer up to a 16x decimation during execution (the number of time samples per packet). To select this, choose a Stokes parameter and add a log 2 of the factor to the mode.
 

src/CLI/ascii_hdr_manager.h

@@ -21,6 +21,18 @@
 
 #endif
 
+// Slow stopping macro (enable from makefile)
+#ifndef __LOFAR_SLEEP
+#define __LOFAR_SLEEP
+
+#ifdef __SLOWDOWN
+#include <unistd.h>
+#define PAUSE sleep(1);
+#else
+#define PAUSE while(0) {};
+#endif
+
+#endif
 
 #ifndef __ASCII_HDR_H
 #define __ASCII_HDR_H

src/CLI/lofar_cli_extractor.c

@@ -12,10 +12,10 @@ void helpMessages() {
 	printf("-m: <numPack>	Number of packets to process in each read request (default: 65536)\n");
 	printf("-u: <numPort>	Number of ports to combine (default: 4)\n");
 	printf("-t: <timeStr>	String of the time of the first requested packet, format YYYY-MM-DDTHH:mm:ss (default: '')\n");
-	printf("-s: <numSec>	Maximum number of seconds to process (default: all)\n");
+	printf("-s: <numSec>	Maximum number of seconds of raw data to extract/process (default: all)\n");
 	printf("-e: <fileName>	Specify a file of events to extract; newline separated start time and durations in seconds. Events must not overlap.\n");
 	printf("-p: <mode>		Processing mode, options listed below (default: 0)\n");
-	printf("-r:		Replay the previous packet when a dropped packet is detected (default: 0 pad)\n");
+	printf("-r:		Replay the previous packet when a dropped packet is detected (default: pad with 0 values)\n");
 	printf("-c:		Change to the alternative clock used for modes 4/6 (160MHz clock) (default: False)\n");
 	printf("-q:		Enable silent mode for the CLI, don't print any information outside of library error messes (default: False)\n");
 	printf("-a: <args>		Call mockHeader with the specific flags to prefix output files with a header (default: False)\n");
@@ -160,7 +160,8 @@ int main(int argc, char  *argv[]) {
 	// processingMode -> N output-files 
 	outputFilesCount = ports;
 	if (processingMode == 2 || processingMode == 11 || processingMode == 21) outputFilesCount = UDPNPOL;
-	else if (processingMode == 10 || processingMode == 20 || processingMode > 99) outputFilesCount = 1;
+	else if (processingMode == 10 || processingMode == 20 || (processingMode > 99 && processingMode < 140)) outputFilesCount = 1;
+	else if (processingMode > 149 && processingMode < 160) outputFilesCount = 4;
 
 	// Sanity check a few inputs
 	if ( (strcmp(inputFormat, "") == 0) || (ports == 0) || (packetsPerIteration < 2)  || (replayDroppedPackets > 1 || replayDroppedPackets < 0) || (processingMode > 1000 || processingMode < 0) || (seconds < 0)) {
@@ -191,7 +192,7 @@ int main(int argc, char  *argv[]) {
 	}
 
 	if (silent == 0) {
-		printf("LOFAR UDP Data extractor (CLI v%.1f, Backend V%.1f)\n\n", VERSIONCLI, VERSION);
+		printf("LOFAR UDP Data extractor (CLI v%.1f, Backend v%.1f)\n\n", VERSIONCLI, VERSION);
 		printf("=========== Given configuration ===========\n");
 		printf("Input File:\t%s\nOutput File: %s\n\n", inputFormat, outputFormat);
 		printf("Packets/Gulp:\t%ld\t\t\tPorts:\t%d\n\n", packetsPerIteration, ports);
@@ -358,7 +359,12 @@ int main(int argc, char  *argv[]) {
 	if (reader == NULL) {
 		fprintf(stderr, "Failed to generate reader. Exiting.\n");
 		return 1;
+	}
 
+	// Sanity check that we were passed the correct clock bit
+	if (((lofar_source_bytes*) &(reader->meta->inputData[0][1]))->clockBit != clock200MHz) {
+		fprintf(stderr, "ERROR: The clock bit of the first packet does not match the clock state given when starting the CLI. Add or remove -c from your command. Exiting.\n");
+		return 1;
 	}
 
 	if (silent == 0) {
@@ -395,7 +401,7 @@ int main(int argc, char  *argv[]) {
 		if (eventCount > 1) 
 			if (silent == 0) {
 				if (eventLoop > 0)  {
-					printf("Completed work for event %d, packet loss for each port during this event was", eventCount -1);
+					printf("Completed work for event %d, packets lost for each port during this event was", eventCount -1);
 					for (int port = 0; port < reader->meta->numPorts; port++) printf(" %ld", eventPacketsLost[port]);
 					printf(".\n\n\n");
 				}

src/CLI/lofar_cli_guppi_raw.c

@@ -177,7 +177,7 @@ int main(int argc, char  *argv[]) {
 
 
 	if (silent == 0) {
-		printf("LOFAR UDP Data extractor (CLI v%.1f, Backend V%.1f)\n\n", VERSIONCLI, VERSION);
+		printf("LOFAR UDP Data extractor (CLI v%.1f, Backend v%.1f)\n\n", VERSIONCLI, VERSION);
 		printf("=========== Given configuration ===========\n");
 		printf("Input File:\t%s\nOutput File: %s\n\n", inputFormat, outputFormat);
 		printf("Packets/Gulp:\t%ld\t\t\tPorts:\t%d\n\n", packetsPerIteration, ports);
@@ -272,7 +272,12 @@ int main(int argc, char  *argv[]) {
 	if (reader == NULL) {
 		fprintf(stderr, "Failed to generate reader. Exiting.\n");
 		return 1;
+	}
 
+	// Sanity check that we were passed the correct clock bit
+	if (((lofar_source_bytes*) &(reader->meta->inputData[0][1]))->clockBit != clock200MHz) {
+		fprintf(stderr, "ERROR: The clock bit of the first packet does not match the clock state given when starting the CLI. Add or remove -c from your command. Exiting.\n");
+		return 1;
 	}
 
 	// Initialise the ASCII header struct if a metadata file was provided

src/CLI/lofar_cli_meta.c

@@ -25,6 +25,8 @@ void processingModes(void) {
 	printf("120: Raw UDP to Stokes U: Form a 32-bit float Stokes U for the input.\n");
 	printf("130: Raw UDP to Stokes V: Form a 32-bit float Stokes V for the input.\n\n");
 
+	printf("150: Raw UDP to Full Stokes: Form a 32-bit float Stokes Vector for the input (I, Q, U, V output files)\n\n");
+
 	printf("Stokes outputs can be decimated in orders of 2, up to 16x by adjusting the last digit of their processing mode.\n");
 	printf("This is handled in orders of two, so 101 will give a Stokes I with 2x decimation, 102, will give 4x, 103 will give 8x and 104 will give 16x.\n");
 }

src/lib/lofar_udp_backends.cpp

@@ -552,3 +552,89 @@ int lofar_udp_raw_udp_stokesV_sum16(lofar_udp_meta *meta) {
 			return 1;
 	}
 }
+
+
+int lofar_udp_raw_udp_full_stokes(lofar_udp_meta *meta) {
+	VERBOSE(if (meta->VERBOSE) printf("Entered C++ call for lofar_udp_raw_udp_full_stokes\n"));
+	switch(meta->inputBitMode) {
+		case 4:
+			fprintf(stderr, "4-bit mode is not yet supported, exiting.\n");
+			return 1;
+		case 8:
+			return lofar_udp_raw_loop<signed char, float, 150>(meta);
+		case 16:
+			return lofar_udp_raw_loop<signed short, float, 150>(meta);
+
+		default:
+			fprintf(stderr, "Unexpected bitmode %d. Exiting.\n", meta->inputBitMode);
+			return 1;
+	}
+}
+
+int lofar_udp_raw_udp_full_stokes_sum2(lofar_udp_meta *meta) {
+	VERBOSE(if (meta->VERBOSE) printf("Entered C++ call for lofar_udp_raw_udp_full_stokes_sum2\n"));
+	switch(meta->inputBitMode) {
+		case 4:
+			fprintf(stderr, "4-bit mode is not yet supported, exiting.\n");
+			return 1;
+		case 8:
+			return lofar_udp_raw_loop<signed char, float, 151>(meta);
+		case 16:
+			return lofar_udp_raw_loop<signed short, float, 151>(meta);
+
+		default:
+			fprintf(stderr, "Unexpected bitmode %d. Exiting.\n", meta->inputBitMode);
+			return 1;
+	}
+}
+
+int lofar_udp_raw_udp_full_stokes_sum4(lofar_udp_meta *meta) {
+	VERBOSE(if (meta->VERBOSE) printf("Entered C++ call for lofar_udp_raw_udp_full_stokes_sum4\n"));
+	switch(meta->inputBitMode) {
+		case 4:
+			fprintf(stderr, "4-bit mode is not yet supported, exiting.\n");
+			return 1;
+		case 8:
+			return lofar_udp_raw_loop<signed char, float, 152>(meta);
+		case 16:
+			return lofar_udp_raw_loop<signed short, float, 152>(meta);
+
+		default:
+			fprintf(stderr, "Unexpected bitmode %d. Exiting.\n", meta->inputBitMode);
+			return 1;
+	}
+}
+
+int lofar_udp_raw_udp_full_stokes_sum8(lofar_udp_meta *meta) {
+	VERBOSE(if (meta->VERBOSE) printf("Entered C++ call for lofar_udp_raw_udp_full_stokes_sum8\n"));
+	switch(meta->inputBitMode) {
+		case 4:
+			fprintf(stderr, "4-bit mode is not yet supported, exiting.\n");
+			return 1;
+		case 8:
+			return lofar_udp_raw_loop<signed char, float, 153>(meta);
+		case 16:
+			return lofar_udp_raw_loop<signed short, float, 153>(meta);
+
+		default:
+			fprintf(stderr, "Unexpected bitmode %d. Exiting.\n", meta->inputBitMode);
+			return 1;
+	}
+}
+
+int lofar_udp_raw_udp_full_stokes_sum16(lofar_udp_meta *meta) {
+	VERBOSE(if (meta->VERBOSE) printf("Entered C++ call for lofar_udp_raw_udp_full_stokes_sum16\n"));
+	switch(meta->inputBitMode) {
+		case 4:
+			fprintf(stderr, "4-bit mode is not yet supported, exiting.\n");
+			return 1;
+		case 8:
+			return lofar_udp_raw_loop<signed char, float, 154>(meta);
+		case 16:
+			return lofar_udp_raw_loop<signed short, float, 154>(meta);
+
+		default:
+			fprintf(stderr, "Unexpected bitmode %d. Exiting.\n", meta->inputBitMode);
+			return 1;
+	}
+}