# -*- coding: utf-8 -*-
"""
Created on Wed Jul 29 13:58:24 2015
v 2018-Jan-22

@author: Jacob Hedelius
"""
#!/bin/python
# Filename readzenofiles.py

__author__ = 'Jacob Hedelius'

def readzenofiles(filename):
    
    
    # This function is meant to read in zeno files in their various formats and write 
    # them to a "structure"/dictionary
    
    # example data
    #ex_filename = 'C:\\Users\\Jacob Hedelius\\Documents\\Research\\python\\example_data\\cn_cmb_zeno.zno'
  #  ex_filename = 'C:\\Users\\Jacob_2\\Documents\\Research\\EM27\\Zeno\\Retrieved_Zeno_Data\\20180121_00_ta.txt'
  #  filename = ex_filename
    
    # import packages
    import numpy as np
    import re
    import egi_subs as egs    
    
    
    # read in file
    fobj = open(filename, 'r') # gets file object for reading
    file_lines = fobj.readlines() #reads in entire file all at once, line by line
    fobj.close()
    
    
    
    # initialize while loop
    data_check = np.zeros([80, 1]) #check vector, assumes there won't be more than 80 lines of header
    # NOTE: for your data files the first column needs be numbers
    i1 = 0 
    i1a = 1
    
    numLines=np.shape(file_lines)[0]
    if numLines>150: #if there are a lot of lines, at least 50 need to be numbers
      rws_num = 50
    elif numLines<10:
      rws_num = 5
    else:
      rws_num = 10 #required number of rows with numbers (in a row) to be accepted as non-header
    
    # find where data starts being printed  (as opposed to just header)
    while np.sum(data_check[i1a:i1]) != rws_num:
                
        line_i1 = re.split('[,|/|:|\n|\t|^M| ]+', file_lines[i1]) #splits line
        line_cln = list(filter(None, line_i1)) #remove empties
        
        line_digs = np.zeros([len(line_cln),1])
        line_digs[:] = np.NAN
        i2 = 0
        for field_i1 in line_cln:
            line_digs[i2] = egs.is_number(field_i1)
            i2 = i2+1
        
        if line_digs[0] == 1:
            data_check[i1] = 1
            
        
        i1 = i1+1
        if i1 > rws_num:
            i1a = i1 - rws_num
        else:
            i1a = i1        
            
    header_pre = re.split('[,|/|:|\n|\r|\t|^M| ]+', file_lines[i1a-1])
    header = list(filter(None, header_pre))
    
    out = {'filename': filename}
    out['header'] = header
    
    if i1a >= 1:
        info = file_lines[0:(i1a-1)]
    
    out['info'] = info
    
    
    
    
    #find index of date/time headers
    UDind = -1
    UTind = -1
    CDind = -1
    CTind = -1
    
    try: UDind = header.index('UTCDate'); #finding if UTCDate header exists and where
    except(ValueError): pass  
    try: UTind = header.index('UTCTime'); #finding if UTCTime header exists and where
    except(ValueError): pass
    try: CDind = header.index('CompDate'); #finding if UTCDate header exists and where
    except(ValueError): pass  
    try: CTind = header.index('CompTime'); #finding if UTCTime header exists and where
    except(ValueError): pass 
    
    
    
    # specify if UTC, local, or CompSrlDate should be used 
    utc_lcl = 0 #default is neither
    if np.all((CDind>-1,CTind>-1)): #if computer time is avaliable (first choice)
      utc_lcl=2 #local
    elif np.all((UDind>-1,UTind>-1)):
      utc_lcl=1 #utc
    
    
    
    # pre-allocate lists that should contain met data
    data_array = {'Pout': [None] * (len(file_lines)-i1a) } #pre-allocate dictionary
    for header_i1 in header:
        if np.any((header_i1 == 'CompDate', header_i1 == 'UTCDate')): 
          if header_i1=='UTCDate':
             pre_UC = 'U' #utc time
          elif header_i1=='CompDate': #comp time
             pre_UC = 'C' #comp time
          
          data_array[pre_UC + 'year'] = [None] * (len(file_lines)-i1a) 
          data_array[pre_UC + 'month'] = [None] * (len(file_lines)-i1a) 
          data_array[pre_UC + 'day'] = [None] * (len(file_lines)-i1a) 
    
        elif np.any((header_i1 == 'CompTime', header_i1 == 'UTCTime')):
            if header_i1=='UTCTime':
                pre_UC = 'U' #utc time
            elif header_i1=='CompTime': #comp time
                pre_UC = 'C' #utc time
            data_array[pre_UC + 'hour'] = [None] * (len(file_lines)-i1a) 
            data_array[pre_UC + 'min'] = [None] * (len(file_lines)-i1a) 
            data_array[pre_UC + 'sec'] = [None] * (len(file_lines)-i1a) 
            
        elif header_i1 == 'Bit':
            data_array[header_i1] = [None] * (len(file_lines)-i1a) #pre-allocate as list, not array
        else:
            data_array[header_i1] = [None] * (len(file_lines)-i1a) #
    
    
    # loop through and save data, row by row into lists (note: this may be somewhat inefficient...)  
    lpi1 = 0
    i1 = 0
    for line_i1 in file_lines:
        if lpi1 >= i1a: #not header
            
            line_sptp = re.split('[,|/|:|\n|\r|\t|^M| ]+', line_i1) #split line
            line_spt = list(filter(None, line_sptp))
        
            
            i2 = 0
            for header_i1 in header: #write individual fields
                if np.any(( np.all((header_i1=='CompDate',utc_lcl==2)), np.all((header_i1=='UTCDate',utc_lcl==1)) )):
                  if header_i1=='UTCDate':
                    pre_UC = 'U' #utc time
                  elif header_i1=='CompDate': #comp time
                    pre_UC = 'C' #comp time
                  
                  data_array[pre_UC + 'year'][i1] = line_spt[i2]
                  data_array[pre_UC + 'month'][i1] = line_spt[i2+1]
                  data_array[pre_UC + 'day'][i1] = line_spt[i2+2]
                  
    
                  i2 = i2 + 2
                elif np.any((np.all((header_i1=='CompTime',utc_lcl==2)), np.all((header_i1=='UTCTime',utc_lcl==1)) )):
                  if header_i1=='UTCTime':
                    pre_UC = 'U' #utc time
                  elif header_i1=='CompTime': #comp time
                    pre_UC = 'C' #comp time
                  
                  data_array[pre_UC + 'hour'][i1] = line_spt[i2]
                  data_array[pre_UC + 'min'][i1] = line_spt[i2+1]
                  data_array[pre_UC + 'sec'][i1] = line_spt[i2+2]
                  i2 = i2 + 2
    
                else: #everything else...
                  data_array[header_i1][i1] = line_spt[i2]
                    
                    
                i2 = i2 + 1
                
            i1 = i1 + 1
            
        lpi1 = lpi1 + 1
    
    
    
    # try to convert lists into numerical numpy arrays if possible
    for param_i1 in data_array:
      i1_ary = np.array(data_array[param_i1]) #attempt to convert to numpy array
      try: i2_ary = i1_ary.astype('float'); verr=0 #try to convert array to numbers
      except(ValueError): verr=1
      
      if verr == 0: #no error
        out[param_i1] = i2_ary #save the converted data
      else:
        out[param_i1] = data_array[param_i1] #save the original list of data
    
    
    out['utc_lcl'] =  utc_lcl #saves the UTC or loval flag
    yr_flds = ['year', 'Uyear', 'Cyear']
    for param_i1 in yr_flds:
      if param_i1 in out:
        out[param_i1][out[param_i1]<100] = out[param_i1][out[param_i1]<100] + 2000 #fixes yy dates
        
    
    
    return out
  
  
## may consider some sort of check at the end to ensure file was read in correctly...










