strip_data.py

import sys
import numpy as np
import scipy as sp
import scipy.signal as ss


DDR_FILLING   = [0xba98,0xfedc]
pixels = 512
TAIL = [0xba98,0xfedc]*16
DEAD = [0xdea1,0xdead,0xdea2,0xdead,0xdea3,0xdead,0xdea4,0xdead,0xdea5,0xdead,0xdea6,0xdead,0xdea7,0xdead,0xdea8,0xdead]

def strip_data(filename, number_pixels, orbits, offset = None):

        ###############----------Read data from memory----------############################################
        np.set_printoptions(threshold='nan')
        # data = np.fromfile(str(filename), dtype=np.uint16)
        if 	offset is None:
        	data = np.memmap( str(filename), dtype=np.dtype('<u2'), mode='r')
        else:
            data = np.memmap( str(filename), dtype=np.dtype('<u2'), offset = (2*pixels*offset),mode='r')


        ###############----------Remove Header----------####################################################

        if (data[0] == 0x1111 and data[15] == 0xF888):
            print ('Removing header .....')
            data = data[16:]
        else:
            print ('No data header....')
            data = data

        ##############--------Search and remove for TAIL--------############################################
        offset = 0
        tailoffset = offset
        while np.any((data[tailoffset : tailoffset + len(TAIL)] != TAIL)):
            tailoffset += 8
            if tailoffset + len(TAIL) > len(data):
                print('Could not find tail')
		       #sys.exit(-1)

        print('Found tail at offset ' + str(tailoffset))

        ##############--------Search and remove for DEAD filling--------##################################
        deadcnt = 0
        deadoffset = tailoffset
        while np.any(data[deadoffset - len(DEAD) : deadoffset] == DEAD):
            deadcnt +=1
            deadoffset -= len(DEAD)

        print('Removed %d bytes of DEAD' %(deadcnt))

        ##############--------Set the pure data --------###################################################

        data = np.frombuffer((data[offset:deadoffset]), dtype = np.uint16)

        #dead = np.where(data == 0xdea1)
        #loc = np.amin(dead)
        #data = data[0:loc]

        ############-------Swap the data bytes and change from 16 bit to 14 bit---------####################

        data = data.byteswap()
        data = data >> 2

        #############---------Index decoder for the jesd data for 16 input mode----------###################
        inner_idx_1 = [23, 22, 7, 6, 21, 20,  5, 4, 19, 18,  3, 2, 17, 16 ,1, 0]        # working one
        inner_idx_2 = [8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31]    # non interleaved

        index_1 = np.zeros([256], dtype=np.int64)
        index_2 = np.zeros([256], dtype=np.int64)
        for i in range(16):
            for j in range(16):
                index_1[i * 16 + j] = inner_idx_1[i] +  j * 32

        for i in range(16):
            for j in range(16):
                index_2[i * 16 + j] = inner_idx_2[i] +  j * 32

        ################------working part---------###########################################################
        #index_interleave = np.ravel(np.column_stack((index_1,index_2)))  #interleaving the pixels, NOTE: interchange index_1 & index_2 if the pixels are swapped
        #k = 32
        #pixel_map = []
        #a = index_interleave

        #for i in range(0, len(a), k):
        #    pixel_map.extend((a[i:i + k])[::-1])
        ######################################################################################################
        l = 16
        adc_2 = []
        b = index_2
        for i in range(0, len(b), l):
            adc_2.extend((b[i:i + l])[::-1])



        #index_interleave = np.concatenate((index_1,adc_2),axis = 0)    # for concatenated pixels

        index_interleave = np.ravel(np.column_stack((index_1,adc_2)))    # for interleaved pixels



        ##############-------------Reshape the data as pixel x orbits-------------------###################
        if (orbits):
            data = data[0:orbits*512]

        data = np.reshape(data,(-1,pixels))


        data = data[:,index_interleave]

        ###### filter out the saw pattern ####################
        data = ss.savgol_filter(data, 21, 3) # window size 51, polynomial order 3

        orb, pix = np.shape(data)

        return (data, orb)