function out = readzenofiles(filename)

% purpose of this function is to read in expected Zeno files
% site with the extension.  This reader is more specialized for these file
% types. 

% v - 2018-Jan-22 - JKH

% example filename
% filename = '../../../../../../oco2-data/tccon/spectra/caltech_em27/i2s_infiles/20140628/ci_cmb_zeno.zno';


%% initial count of number of lines
fid = fopen(filename, 'rb');
% Get file size.
fseek(fid, 0, 'eof');
fileSize = ftell(fid);
frewind(fid);
% Read the whole file.
data = fread(fid, fileSize, 'uint8');
% Count number of line-feeds.
numLines = sum(data == 10);
fclose(fid);

%% reopen file, for actual reading
fileID = fopen(filename);

header_int = cell(80,1); %pre-allocates for speed on the guess that there won't be more than 80 lines of header

data_check = zeros(size(header_int));
i1 = 1; %initialize while loop
i1a = 1; %second (starting) index
if numLines>150 %if there are a lot of lines, at least 50 need to be numbers
  rws_num = 50;
elseif numLines<10
  rws_num = 5;
else
  rws_num = 10;
end

while sum(data_check(i1a:i1)) ~= rws_num %assumes that if there has been 10 rows of numbers then data has started,
%     note that this has the disadvantage of requiring at least 10 rows
    
    header_int{i1} = fgetl(fileID);
    
    line_i1 = textscan(header_int{i1}, '%s', 'delimiter', {',','/',':',' ','\t','^M'}, 'multipleDelimsAsOne', 1); %reads all as strings initially
    col_num = str2double(line_i1{1,1}); %attempts to convert line to numbers
    
    data_check(i1) = all(~isnan(col_num(1:2))); %if it is NaN (strings) in the first 2 columns, it's likely header text
    
    
    i1 = i1+1;
    if i1 <= rws_num
        i1a = i1;
    else
        i1a = i1- rws_num;
    end
end
% fileID = fclose('all'); %close file (so it can be reopened and re-read)
close('all');

header = textscan(header_int{i1a-1}, '%s', 'delimiter', {',','/',':',' ','\t','^M'}, 'multipleDelimsAsOne', 1);
header = header{1,1}'; %creates line of data units header

header_int = header_int(1:(i1a-2),1);

%% read in and attempt to convert to numbers
scan_format = repmat('%s',1,length(line_i1{1,1})); %sets-up to read entire line in as strings (convert to number later)

    
fileID = fopen(filename);
for i1 = 1:(i1a-1) %retrims off first rows of file that contain header without removing the data
    fgetl(fileID);
end

filedata_raw = textscan(fileID, scan_format, 'delimiter', {',','/',':',' ','\t','^M', '\n', '\r'}, 'multipleDelimsAsOne', 1); % filedataraw_a = filedata_raw;
fclose('all');


fdata_num = cell(1,size(filedata_raw,2)); %pre-allocates for converting data to numeric where possible
for i1 = 1:length(fdata_num) %for each column of data
  Stmp = sprintf('%s*', filedata_raw{i1}{:}); %temporary string of data, with "*" as delimiters
  [col_i1,n_i1,err_msg]=sscanf(Stmp, '%f*'); %attempt to convert the data to strings (this is faster than str2double)
  if n_i1==length(filedata_raw{i1}) && isempty(err_msg)
    col_i1(col_i1 == -9999) = NaN; %removes bad data values, sets as NaN
    col_i1(col_i1 == -10000) = NaN;
    fdata_num{i1} = col_i1;
  else %there was an error, likely column contains strings so it cannot convert to numbers
    fdata_num{i1} = filedata_raw{i1};
  end
end

%% setup the header (fix illegal characters) and start saving to output structure
header_org = header; %copy the header
header = cellfun(@(x) strrep(x, '(', '_'), header, 'UniformOutput',  false); %replace parenthesis
header = cellfun(@(x) strrep(x, ')', '_'), header, 'UniformOutput',  false);
header = cellfun(@(x) strrep(x, '-', '_'), header, 'UniformOutput',  false); %replace underscore


out.filename = filename;
out.info = header_int;
out.hdr_org = header_org;
out.header = header;
out.data = filedata_raw;

%% output variables
utc_lcl = 0; %default is neither
if any(not(cellfun(@isempty, strfind(header, 'CompDate')))) && any(not(cellfun(@isempty, strfind(header, 'CompTime'))))%if computer time is avaliable (first choice)
  utc_lcl=2; %local
elseif any(not(cellfun(@isempty, strfind(header, 'UTCDate')))) && any(not(cellfun(@isempty, strfind(header, 'UTCTime'))))%
  utc_lcl=1; %utc
end
out.utc_lcl=utc_lcl; %output what type of time parameter

i1_ind = 1;
for i1 = 1:length(header)
%     idx = find(strcmp([C{:}], 'a'))
    if all(cellfun(@isempty, strfind({'UTCDate','UTCTime','CompDate','CompTime'},header{i1})))  %if header does not match any of these strings
        out.(header{i1}) = fdata_num{i1_ind};
        i1_ind = i1_ind+1;
    elseif (isequal(header{i1}, 'UTCDate') && utc_lcl==1) || (isequal(header{i1}, 'CompDate') && utc_lcl==2)
        if isequal(header{i1}, 'UTCDate')
            pre_UC = 'U'; %utc time
        elseif isequal(header{i1}, 'CompDate')
            pre_UC = 'C'; %comp time
        end
        out.([pre_UC, 'year']) = fdata_num{i1_ind};
        if out.([pre_UC, 'year'])(1) < 100 %years are in YY format instead of YYYY
            out.([pre_UC, 'year']) = out.([pre_UC, 'year']) + 2000;
        end
        
        out.([pre_UC, 'month']) = fdata_num{i1_ind+1};
        out.([pre_UC, 'day']) = fdata_num{i1_ind+2};
        i1_ind = i1_ind+3;
    elseif (isequal(header{i1}, 'UTCTime') && utc_lcl==1) || (isequal(header{i1}, 'CompTime') && utc_lcl==2)
        if isequal(header{i1}, 'UTCTime')
            pre_UC = 'U'; %utc time
        elseif isequal(header{i1}, 'CompTime')
            pre_UC = 'C'; %comp time
        end
        out.([pre_UC, 'hour']) = fdata_num{i1_ind};
        out.([pre_UC, 'min']) = fdata_num{i1_ind+1};
        out.([pre_UC, 'sec']) = fdata_num{i1_ind+2};
        i1_ind = i1_ind+3;
        
    end
    
end



%% Compare size of file with what's been read in.  If they do not match, there's an error.

length_read = (i1a-1) + length(out.([pre_UC, 'year'])); %rows of header + rows of data
if abs(numLines-length_read) > 1
 warning('Zeno file does not appear to be read in correctly')
end






